From 79c434c0d9560474d08361f91b4425ec2cbf41a5 Mon Sep 17 00:00:00 2001 From: Andrei Aristarkhov Date: Mon, 28 Sep 2026 15:23:54 -0700 Subject: [PATCH 001/101] build: fetch SPDX license list data as a third-party dependency Pins spdx/license-list-data at commit 31ba1a50 (v3.29.0), so that an SBOM collector can name a dependency's license by matching the text that dependency ships against the SPDX reference texts. Nothing from the checkout is redistributed, and it builds nothing. Upstream publishes every license in several formats, so the download command clones with --filter=tree:0, sparse-checks out json/, and detaches at the pinned commit: 27 MB instead of 405 MB, and a no-op on reconfigure once that commit is already checked out. Pinned by commit rather than by tag, so nothing upstream can move underneath a build. Signed-off-by: Andrei Aristarkhov --- deps/README.md | 9 ++++ deps/third_party/CMakeLists.txt | 33 +++++++++++++++ .../third_party/fetch_license_list_data.cmake | 41 +++++++++++++++++++ 3 files changed, 83 insertions(+) create mode 100644 deps/third_party/fetch_license_list_data.cmake diff --git a/deps/README.md b/deps/README.md index aa8ddf9042..1a99df6766 100644 --- a/deps/README.md +++ b/deps/README.md @@ -95,6 +95,15 @@ machine and are located via `find_package`. - **Build**: Header-only library - **License**: BSD-style +### SPDX license list data +- **Source**: https://github.com/spdx/license-list-data.git +- **Version**: 31ba1a50e5397e00a304dbadc76531740e89ee48 (v3.29.0) +- **Purpose**: reference license texts. The SBOM collector names a dependency's + license by matching the text that dependency ships against these; nothing from + this checkout is redistributed. +- **Build**: data only, no build step. +- **License**: CC0-1.0 + ### GLM - **Source**: https://github.com/g-truc/glm.git - **Version**: 1.0.1 diff --git a/deps/third_party/CMakeLists.txt b/deps/third_party/CMakeLists.txt index 4a809c948c..0795f0be1a 100644 --- a/deps/third_party/CMakeLists.txt +++ b/deps/third_party/CMakeLists.txt @@ -278,3 +278,36 @@ if(BUILD_VIZ AND BUILD_PYTHON_BINDINGS) include("${CMAKE_CURRENT_LIST_DIR}/Mujoco.cmake") endif() + +# ============================================================================== +# SPDX license list data +# ============================================================================== +# Reference license texts. The SBOM collector names a dependency's license by +# matching the text that dependency ships against these; nothing from this +# checkout is redistributed, and it builds nothing. Pinned by commit, so no tag +# upstream can move underneath a build. +find_package(Git REQUIRED) + +set(SPDX_LICENSE_DATA_COMMIT "31ba1a50e5397e00a304dbadc76531740e89ee48") # v3.29.0 +set(SPDX_LICENSE_DATA_SOURCE_DIR "${FETCHCONTENT_BASE_DIR}/license-list-data-src") + +message(STATUS "Fetching SPDX license list data from GitHub...") +FetchContent_Declare( + license_list_data + SOURCE_DIR "${SPDX_LICENSE_DATA_SOURCE_DIR}" + DOWNLOAD_COMMAND + "${CMAKE_COMMAND}" + "-DGIT_EXECUTABLE=${GIT_EXECUTABLE}" + "-DSPDX_REPOSITORY=https://github.com/spdx/license-list-data.git" + "-DSPDX_COMMIT=${SPDX_LICENSE_DATA_COMMIT}" + "-DSPDX_SOURCE_DIR=${SPDX_LICENSE_DATA_SOURCE_DIR}" + -P "${CMAKE_CURRENT_LIST_DIR}/fetch_license_list_data.cmake" +) +FetchContent_MakeAvailable(license_list_data) + +# What the SBOM collector reads, and what the sbom tests run against. +set(SPDX_LICENSE_DATA_JSON_DIR "${SPDX_LICENSE_DATA_SOURCE_DIR}/json" + CACHE INTERNAL "SPDX reference license texts for the SBOM collector") +if(NOT EXISTS "${SPDX_LICENSE_DATA_JSON_DIR}/licenses.json") + message(FATAL_ERROR "SPDX license list data is missing json/licenses.json after fetch") +endif() diff --git a/deps/third_party/fetch_license_list_data.cmake b/deps/third_party/fetch_license_list_data.cmake new file mode 100644 index 0000000000..7678eacc5a --- /dev/null +++ b/deps/third_party/fetch_license_list_data.cmake @@ -0,0 +1,41 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Narrowed to the JSON the SBOM collector reads: upstream publishes every license +# in several formats, none of which anything here consumes. + +foreach(_required GIT_EXECUTABLE SPDX_REPOSITORY SPDX_COMMIT SPDX_SOURCE_DIR) + if(NOT DEFINED ${_required}) + message(FATAL_ERROR "${_required} is required") + endif() +endforeach() + +if(EXISTS "${SPDX_SOURCE_DIR}/.git") + execute_process( + COMMAND "${GIT_EXECUTABLE}" rev-parse HEAD + WORKING_DIRECTORY "${SPDX_SOURCE_DIR}" + OUTPUT_VARIABLE _current_commit + OUTPUT_STRIP_TRAILING_WHITESPACE + RESULT_VARIABLE _rev_parse_result + ) + if(_rev_parse_result EQUAL 0 AND _current_commit STREQUAL SPDX_COMMIT) + return() + endif() +endif() + +file(REMOVE_RECURSE "${SPDX_SOURCE_DIR}") +execute_process( + COMMAND "${GIT_EXECUTABLE}" clone --quiet --filter=tree:0 --no-checkout + "${SPDX_REPOSITORY}" "${SPDX_SOURCE_DIR}" + COMMAND_ERROR_IS_FATAL ANY +) +execute_process( + COMMAND "${GIT_EXECUTABLE}" sparse-checkout set "json/details" "json/licenses.json" + WORKING_DIRECTORY "${SPDX_SOURCE_DIR}" + COMMAND_ERROR_IS_FATAL ANY +) +execute_process( + COMMAND "${GIT_EXECUTABLE}" checkout --quiet --detach "${SPDX_COMMIT}" + WORKING_DIRECTORY "${SPDX_SOURCE_DIR}" + COMMAND_ERROR_IS_FATAL ANY +) From f867de7042fdb09df08ba8dd6a6cd84b124d594e Mon Sep 17 00:00:00 2001 From: Andrei Aristarkhov Date: Tue, 29 Sep 2026 17:34:09 -0700 Subject: [PATCH 002/101] build: separate third-party OpenXR extension headers from NVIDIA's deps/cloudxr/openxr_extensions held seven headers, two of them written by other people: openxr_extension_helpers.h (The Khronos Group, Apache-2.0) and XR_MNDX_xdev_space.h (Collabora, BSL-1.0). Both compile into shipped extension modules, so this project redistributes them under those terms, and nothing in the layout said so. Move the two into deps/third_party/openxr_extensions behind its own interface target, which Teleop::openxr_extensions carries -- so every consumer keeps linking one target and every #include is unchanged. No header is modified. The moves are pure renames, and the NVIDIA headers keep "openxr_extension_helpers.h" in quotes: a quoted include falls back to the include path once the file is no longer beside them. Those headers also ship verbatim inside the CloudXR SDK tarball, so editing them here would make our copies diverge from upstream's. Signed-off-by: Andrei Aristarkhov --- deps/README.md | 12 ++++++++++++ deps/cloudxr/openxr_extensions/CMakeLists.txt | 9 ++++++++- deps/third_party/CMakeLists.txt | 3 +++ .../openxr_extensions/.clang-format | 2 ++ .../openxr_extensions/CMakeLists.txt | 18 ++++++++++++++++++ .../openxr_extensions/XR_MNDX_xdev_space.h | 0 .../openxr_extension_helpers.h | 0 7 files changed, 43 insertions(+), 1 deletion(-) create mode 100644 deps/third_party/openxr_extensions/.clang-format create mode 100644 deps/third_party/openxr_extensions/CMakeLists.txt rename deps/{cloudxr => third_party}/openxr_extensions/XR_MNDX_xdev_space.h (100%) rename deps/{cloudxr => third_party}/openxr_extensions/openxr_extension_helpers.h (100%) diff --git a/deps/README.md b/deps/README.md index 1a99df6766..b4b08ce8ab 100644 --- a/deps/README.md +++ b/deps/README.md @@ -95,6 +95,18 @@ machine and are located via `find_package`. - **Build**: Header-only library - **License**: BSD-style +### OpenXR extension headers (vendored) +- **Location**: `third_party/openxr_extensions/` +- **Contents**: `openxr_extension_helpers.h` (The Khronos Group Inc., Apache-2.0) + and `XR_MNDX_xdev_space.h` (Collabora, Ltd., BSL-1.0) +- **Purpose**: OpenXR extension declarations this project compiles against and + therefore redistributes in the wheel. Checked in rather than fetched, so they + sit here instead of beside NVIDIA's own extension headers in + `cloudxr/openxr_extensions/`. +- **Build**: header-only `Teleop::openxr_extensions_upstream`, carried by + `Teleop::openxr_extensions` so consumers link one target. +- **License**: Apache-2.0 and BSL-1.0 (see `LICENSES/`) + ### SPDX license list data - **Source**: https://github.com/spdx/license-list-data.git - **Version**: 31ba1a50e5397e00a304dbadc76531740e89ee48 (v3.29.0) diff --git a/deps/cloudxr/openxr_extensions/CMakeLists.txt b/deps/cloudxr/openxr_extensions/CMakeLists.txt index 3d7b69f902..aedea5d8d7 100644 --- a/deps/cloudxr/openxr_extensions/CMakeLists.txt +++ b/deps/cloudxr/openxr_extensions/CMakeLists.txt @@ -1,9 +1,16 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 +# NVIDIA's own OpenXR extension headers. The ones written by Khronos and +# Collabora live in deps/third_party/openxr_extensions; this target carries them +# so that linking Teleop::openxr_extensions still reaches every extension header. add_library(teleop_openxr_extensions INTERFACE) add_library(Teleop::openxr_extensions ALIAS teleop_openxr_extensions) target_include_directories(teleop_openxr_extensions INTERFACE $ ) + +target_link_libraries(teleop_openxr_extensions INTERFACE + Teleop::openxr_extensions_upstream +) diff --git a/deps/third_party/CMakeLists.txt b/deps/third_party/CMakeLists.txt index 0795f0be1a..54d8453e3c 100644 --- a/deps/third_party/CMakeLists.txt +++ b/deps/third_party/CMakeLists.txt @@ -5,6 +5,9 @@ cmake_minimum_required(VERSION 3.20) include(FetchContent) +# Vendored third-party sources, checked in rather than fetched. +add_subdirectory(openxr_extensions) + # ============================================================================== # CMake sanitizers # ============================================================================== diff --git a/deps/third_party/openxr_extensions/.clang-format b/deps/third_party/openxr_extensions/.clang-format new file mode 100644 index 0000000000..9d159247d5 --- /dev/null +++ b/deps/third_party/openxr_extensions/.clang-format @@ -0,0 +1,2 @@ +DisableFormat: true +SortIncludes: false diff --git a/deps/third_party/openxr_extensions/CMakeLists.txt b/deps/third_party/openxr_extensions/CMakeLists.txt new file mode 100644 index 0000000000..3065fa186b --- /dev/null +++ b/deps/third_party/openxr_extensions/CMakeLists.txt @@ -0,0 +1,18 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# OpenXR extension headers written by other people, vendored here verbatim: +# +# openxr_extension_helpers.h The Khronos Group Inc. Apache-2.0 +# XR_MNDX_xdev_space.h Collabora, Ltd. BSL-1.0 +# +# Kept apart from deps/cloudxr/openxr_extensions, which holds NVIDIA's own, so +# that what this project redistributes under someone else's terms is visible +# from the directory layout rather than only from the file headers. + +add_library(teleop_openxr_extensions_upstream INTERFACE) +add_library(Teleop::openxr_extensions_upstream ALIAS teleop_openxr_extensions_upstream) + +target_include_directories(teleop_openxr_extensions_upstream INTERFACE + $ +) diff --git a/deps/cloudxr/openxr_extensions/XR_MNDX_xdev_space.h b/deps/third_party/openxr_extensions/XR_MNDX_xdev_space.h similarity index 100% rename from deps/cloudxr/openxr_extensions/XR_MNDX_xdev_space.h rename to deps/third_party/openxr_extensions/XR_MNDX_xdev_space.h diff --git a/deps/cloudxr/openxr_extensions/openxr_extension_helpers.h b/deps/third_party/openxr_extensions/openxr_extension_helpers.h similarity index 100% rename from deps/cloudxr/openxr_extensions/openxr_extension_helpers.h rename to deps/third_party/openxr_extensions/openxr_extension_helpers.h From d70230bf0b7a009a9233d2acbb89f1b25d91b65d Mon Sep 17 00:00:00 2001 From: Andrei Aristarkhov Date: Wed, 30 Sep 2026 16:41:10 -0700 Subject: [PATCH 003/101] build: generate type stubs into the build tree, then stage them pybind11-stubgen wrote .pyi straight into the staging tree, which left them as the only files in the wheel with no build-tree origin. Every other staged file has one -- authored .py resolve to src/python/isaaccapture, extension modules to their CMake link output, _generated_tracker_exports.py to build/generated/trackers -- because staging is a copy destination rather than somewhere a build step writes for the first time. Generate into build/generated/stubs/$ and copy from there, matching what stage_generated_tracker_exports already does. generate_stubs.py now takes the import path and the output path separately: it still imports the module from the staged tree, since that is where the extension modules are, but no longer writes back into it. Signed-off-by: Andrei Aristarkhov --- src/core/python/CMakeLists.txt | 13 ++++++++++-- src/core/python/generate_stubs.py | 35 ++++++++++++++++++------------- 2 files changed, 31 insertions(+), 17 deletions(-) diff --git a/src/core/python/CMakeLists.txt b/src/core/python/CMakeLists.txt index 4034dfc860..1685616ef6 100644 --- a/src/core/python/CMakeLists.txt +++ b/src/core/python/CMakeLists.txt @@ -70,6 +70,11 @@ endif() set(STUBGEN_SCRIPT "${CMAKE_CURRENT_SOURCE_DIR}/generate_stubs.py") set(STUBGEN_PYPROJECT "${CMAKE_CURRENT_SOURCE_DIR}/stubgen_pyproject.toml") set(STUBGEN_PACKAGE_DIR "${CMAKE_BINARY_DIR}/python_package/$") +# Stubs are generated here and copied into the staging tree below. Writing them +# straight into staging would leave them with no build-tree origin, and the file +# a wheel ships must always be traceable to one -- staging is a copy destination, +# never the only place a file exists. +set(STUBGEN_OUTPUT_DIR "${CMAKE_BINARY_DIR}/generated/stubs/$") set(PY_TYPED_MARKER "${STUBGEN_PACKAGE_DIR}/isaaccapture/py.typed") set(STUBGEN_MODULES @@ -101,7 +106,7 @@ foreach(_module IN LISTS STUBGEN_MODULES) list(APPEND STUBGEN_COMMANDS COMMAND ${CMAKE_COMMAND} -E env --unset=VIRTUAL_ENV uv run --project "${CMAKE_BINARY_DIR}/stubgen" --python ${ISAAC_TELEOP_PYTHON_VERSION} - python "${STUBGEN_SCRIPT}" ${_module} "${STUBGEN_PACKAGE_DIR}" + python "${STUBGEN_SCRIPT}" ${_module} "${STUBGEN_PACKAGE_DIR}" "${STUBGEN_OUTPUT_DIR}" ) endforeach() @@ -114,10 +119,14 @@ add_custom_command( COMMAND ${CMAKE_COMMAND} -E copy "${STUBGEN_PYPROJECT}" "${CMAKE_BINARY_DIR}/stubgen/pyproject.toml" # Generate stubs for each pybind11 module using the stubgen pyproject.toml + COMMAND ${CMAKE_COMMAND} -E make_directory "${STUBGEN_OUTPUT_DIR}/isaaccapture" ${STUBGEN_COMMANDS} # Create py.typed marker file (PEP 561) - also serves as build stamp - COMMAND ${CMAKE_COMMAND} -E touch "${PY_TYPED_MARKER}" + COMMAND ${CMAKE_COMMAND} -E touch "${STUBGEN_OUTPUT_DIR}/isaaccapture/py.typed" + + # Stage what was generated. Last, so PY_TYPED_MARKER stamps a complete copy. + COMMAND ${CMAKE_COMMAND} -E copy_directory "${STUBGEN_OUTPUT_DIR}" "${STUBGEN_PACKAGE_DIR}" DEPENDS python_package "${STUBGEN_SCRIPT}" "${STUBGEN_PYPROJECT}" $ $ diff --git a/src/core/python/generate_stubs.py b/src/core/python/generate_stubs.py index 676c06a0ed..98d1ab250e 100644 --- a/src/core/python/generate_stubs.py +++ b/src/core/python/generate_stubs.py @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 """Generate Python stub file (.pyi) for a single Isaac Teleop pybind11 module. @@ -6,10 +6,10 @@ This script uses pybind11-stubgen's Python API to generate type stubs for IDE intellisense. Usage: - python generate_stubs.py + python generate_stubs.py Example: - python generate_stubs.py isaaccapture.deviceio._deviceio /path/to/python_package + python generate_stubs.py isaaccapture.deviceio._deviceio build/python_package/Release build/generated/stubs """ import sys @@ -27,18 +27,20 @@ def _err(message: str) -> None: print(message, file=sys.stderr) -def generate_stub(module_name: str, package_dir: Path) -> bool: +def generate_stub(module_name: str, import_dir: Path, output_dir: Path) -> bool: """Generate stub file for a single pybind11 module. Args: module_name: Fully qualified module name (e.g., "isaaccapture.deviceio._deviceio") - package_dir: Path to the python_package directory containing isaaccapture. + import_dir: Staged package tree to import the module from. + output_dir: Where to write the .pyi. Kept out of the staged tree so the + stub has a build-tree origin like every other generated file. Returns: True if successful, False otherwise. """ - # Add package_dir to sys.path so pybind11-stubgen can import the module - sys.path.insert(0, str(package_dir)) + # Import from the staged tree; write somewhere else entirely. + sys.path.insert(0, str(import_dir)) try: from pybind11_stubgen import ( @@ -59,7 +61,7 @@ def generate_stub(module_name: str, package_dir: Path) -> bool: # Configure stubgen using the proper API args = CLIArgs( module_name=module_name, - output_dir=str(package_dir), + output_dir=str(output_dir), root_suffix="", ignore_invalid_expressions=None, ignore_invalid_identifiers=None, @@ -111,21 +113,24 @@ def generate_stub(module_name: str, package_dir: Path) -> bool: def main() -> int: """Main entry point.""" - if len(sys.argv) != 3: - _err(f"Usage: {sys.argv[0]} ") + if len(sys.argv) != 4: + _err(f"Usage: {sys.argv[0]} ") _err( - f"Example: {sys.argv[0]} isaaccapture.deviceio._deviceio /path/to/python_package" + f"Example: {sys.argv[0]} isaaccapture.deviceio._deviceio " + "build/python_package/Release build/generated/stubs" ) return 1 module_name = sys.argv[1] - package_dir = Path(sys.argv[2]).resolve() + import_dir = Path(sys.argv[2]).resolve() + output_dir = Path(sys.argv[3]).resolve() - if not package_dir.exists(): - _err(f"Error: Package directory does not exist: {package_dir}") + if not import_dir.exists(): + _err(f"Error: Import directory does not exist: {import_dir}") return 1 + output_dir.mkdir(parents=True, exist_ok=True) - if generate_stub(module_name, package_dir): + if generate_stub(module_name, import_dir, output_dir): return 0 else: _err("Stub generation failed.") From 482a7effcf6a2948a55823ab5ada196c257c9c8a Mon Sep 17 00:00:00 2001 From: Andrei Aristarkhov Date: Wed, 30 Sep 2026 15:12:04 -0700 Subject: [PATCH 004/101] build: publish a contents SBOM and license evidence with every wheel Each published wheel gains an SPDX 2.3 inventory of what it redistributes under `.dist-info/sboms/`, the license text behind every obligation under `.dist-info/licenses/`, and sidecar reports attached to tagged releases. The inventory is discovered, not declared. `scripts/sbom` reads what the build left behind -- CMake's file API codemodel, the `_deps/*-src` checkouts, the archives under `deps/`, the working tree -- so a dependency that builds is a dependency that appears, and there is no registry to keep in step with `CMakeLists.txt`. Licenses are named by matching each component's own text against `spdx/license-list-data`, which `deps/third_party` fetches at a pinned commit; an unmatched text becomes a `LicenseRef-` carrying the text verbatim rather than a guess. Two conditions fail the Release build instead of shipping a gap: a wheel member the build cannot explain, and a redistributed component with no license text. Requires CMake 3.27 for `cmake_file_api()`, which lets the codemodel query live in `CMakeLists.txt` rather than in whatever invoked cmake. Reading a generator's own output instead would only work under Makefiles -- Ninja writes no `link.txt`. The collector is a project of its own so its pins live in one place, and is run as `uv run --project scripts/sbom isaaccapture-sbom ...`. Formats it would otherwise parse by hand come from the packages that own them: installer, packaging, license-expression, packageurl-python, pyelftools. Consumer-facing detail is in docs/source/references/sbom.rst. Signed-off-by: Andrei Aristarkhov --- .github/workflows/build-ubuntu.yml | 233 ++++++ .gitignore | 2 + AGENTS.md | 23 + CMakeLists.txt | 11 +- deps/README.md | 2 +- docs/source/index.rst | 1 + docs/source/references/sbom.rst | 255 +++++++ pyproject.toml | 5 +- scripts/sbom/pyproject.toml | 41 ++ scripts/sbom/sbom/__init__.py | 9 + scripts/sbom/sbom/build.py | 265 +++++++ scripts/sbom/sbom/cli.py | 187 +++++ scripts/sbom/sbom/discovery.py | 799 +++++++++++++++++++++ scripts/sbom/sbom/document.py | 349 +++++++++ scripts/sbom/sbom/elf.py | 68 ++ scripts/sbom/sbom/evidence.py | 297 ++++++++ scripts/sbom/sbom/inventory.py | 359 +++++++++ scripts/sbom/sbom/licensing.py | 505 +++++++++++++ scripts/sbom/sbom/report.py | 215 ++++++ scripts/sbom/sbom/validate.py | 305 ++++++++ scripts/sbom/sbom/wheelfile.py | 280 ++++++++ src/core/python/CMakeLists.txt | 4 + src/core/python/pyproject.toml.in | 7 +- tests/python/CMakeLists.txt | 3 + tests/python/sbom/CMakeLists.txt | 31 + tests/python/sbom/conftest.py | 137 ++++ tests/python/sbom/pyproject.toml | 23 + tests/python/sbom/synth.py | 513 +++++++++++++ tests/python/sbom/test_discovery.py | 190 +++++ tests/python/sbom/test_elf.py | 72 ++ tests/python/sbom/test_gates.py | 283 ++++++++ tests/python/sbom/test_licensing.py | 251 +++++++ tests/python/sbom/test_spdx_conformance.py | 34 + tests/python/sbom/test_wheel_sbom.py | 251 +++++++ 34 files changed, 6005 insertions(+), 5 deletions(-) create mode 100644 docs/source/references/sbom.rst create mode 100644 scripts/sbom/pyproject.toml create mode 100644 scripts/sbom/sbom/__init__.py create mode 100644 scripts/sbom/sbom/build.py create mode 100644 scripts/sbom/sbom/cli.py create mode 100644 scripts/sbom/sbom/discovery.py create mode 100644 scripts/sbom/sbom/document.py create mode 100644 scripts/sbom/sbom/elf.py create mode 100644 scripts/sbom/sbom/evidence.py create mode 100644 scripts/sbom/sbom/inventory.py create mode 100644 scripts/sbom/sbom/licensing.py create mode 100644 scripts/sbom/sbom/report.py create mode 100644 scripts/sbom/sbom/validate.py create mode 100644 scripts/sbom/sbom/wheelfile.py create mode 100644 tests/python/sbom/CMakeLists.txt create mode 100644 tests/python/sbom/conftest.py create mode 100644 tests/python/sbom/pyproject.toml create mode 100644 tests/python/sbom/synth.py create mode 100644 tests/python/sbom/test_discovery.py create mode 100644 tests/python/sbom/test_elf.py create mode 100644 tests/python/sbom/test_gates.py create mode 100644 tests/python/sbom/test_licensing.py create mode 100644 tests/python/sbom/test_spdx_conformance.py create mode 100644 tests/python/sbom/test_wheel_sbom.py diff --git a/.github/workflows/build-ubuntu.yml b/.github/workflows/build-ubuntu.yml index fcd3ce8ad1..28444c7deb 100644 --- a/.github/workflows/build-ubuntu.yml +++ b/.github/workflows/build-ubuntu.yml @@ -282,6 +282,73 @@ jobs: mv install/wheels-repaired/*.whl install/wheels/ rmdir install/wheels-repaired || true + # Runs between auditwheel repair and every upload, because the document + # describes the finished wheel and the wheel gains files here. + # The inventory is discovered from this build tree -- CMake's file API + # codemodel, the fetched sources -- so nothing has to be declared anywhere. + - name: Package licenses and contents SBOM into wheels (Release only) + if: matrix.build_type == 'Release' + run: | + set -euo pipefail + shopt -s nullglob + args=() + for wheel in install/wheels/*.whl; do args+=(--wheel "${wheel}"); done + if (( ${#args[@]} == 0 )); then + echo "No wheels to describe" + exit 1 + fi + uv run --project scripts/sbom --python ${{ matrix.python_version }} \ + isaaccapture-sbom build \ + --build-dir "${BUILD_DIR}" "${args[@]}" --out-dir sbom + + - name: Check wheel SBOMs (Release only) + if: matrix.build_type == 'Release' + run: | + set -euo pipefail + shopt -s nullglob + for wheel in install/wheels/*.whl; do + stem="$(basename "${wheel}" .whl)" + uv run --project scripts/sbom --python ${{ matrix.python_version }} \ + isaaccapture-sbom check --wheel "${wheel}" \ + --manifest sbom/manifest.json \ + --build-evidence "sbom/${stem}.build-evidence.json" + done + # Third-party conformance, from the collector's own dev pins. + for document in sbom/*.spdx.json; do + uv run --project scripts/sbom --python ${{ matrix.python_version }} \ + --extra dev pyspdxtools -i "${document}" + done + + - name: Install and import the finished wheel (Release only) + if: matrix.build_type == 'Release' + run: | + set -euo pipefail + shopt -s nullglob + wheels=(install/wheels/*.whl) + # uv pip install re-checks every RECORD digest, so this doubles as proof + # that the rewrite left an installable wheel behind. --find-links resolves + # the isaacteleop transition wheel built alongside it; without it uv goes + # to PyPI for a dependency this build just produced. + uv venv --python ${{ matrix.python_version }} .venv-wheel-check + uv pip install --python .venv-wheel-check \ + --find-links install/wheels "${wheels[@]}" + .venv-wheel-check/bin/python -c "import isaaccapture; print('imported', isaaccapture.__version__)" + # The consumer-side check, run exactly as the published instructions do. + for wheel in "${wheels[@]}"; do + uv run --project scripts/sbom --python ${{ matrix.python_version }} \ + isaaccapture-sbom check --wheel "${wheel}" + done + + - name: Summarize wheel evidence (Release only) + if: matrix.build_type == 'Release' + run: | + set -euo pipefail + { + echo "## Wheel SBOM and license evidence" + echo + cat sbom/*.licenses.md + } >> "${GITHUB_STEP_SUMMARY}" + - name: Create tarball to preserve permissions if: matrix.build_type == 'Release' run: tar -cvf isaacteleop-install.tar -C install . @@ -302,6 +369,15 @@ jobs: if-no-files-found: error retention-days: 7 + - name: Upload wheel SBOM sidecars + if: matrix.build_type == 'Release' + uses: actions/upload-artifact@v6 + with: + name: isaaccapture-sbom-${{ matrix.arch }}-py${{ matrix.python_version }} + path: sbom/ + if-no-files-found: error + retention-days: 7 + pip-editable-install: # Validates the scikit-build-core (PEP 517/660) from-source install path added # for issue #735: an sdist build and `pip install -e .` over the root @@ -1023,6 +1099,100 @@ jobs: echo "All wheel dependency jobs completed successfully" + publish-sbom-evidence: # durable copies attached to the GitHub release + # Actions artifacts expire in days; a tagged release is where the evidence a + # consumer is told to fetch actually lives. Wheels keep their own copy + # embedded either way, so this job publishes, it does not gate. + runs-on: ubuntu-22.04 + needs: + - wheel-gate + permissions: + contents: write + timeout-minutes: 20 + + # `!cancelled()` for the reason spelled out on `publish-wheel`: a skip + # anywhere upstream of `wheel-gate` reaches here too, and the result check + # is what should decide. + if: >- + !cancelled() + && needs.wheel-gate.result == 'success' + && github.repository_id == '1092046207' + && startsWith(github.ref, 'refs/tags/v') + + steps: + - name: Checkout code + uses: actions/checkout@v6 + with: + fetch-depth: 1 + persist-credentials: false + + - name: Install uv + uses: ./.github/actions/setup-uv + + - name: Download wheel artifacts + uses: actions/download-artifact@v7 + with: + pattern: isaacteleop-wheels-* + merge-multiple: true + path: wheels + + - name: Download wheel SBOM sidecars + uses: actions/download-artifact@v7 + with: + pattern: isaaccapture-sbom-* + path: sbom-sidecars + + - name: Assemble and verify the release evidence set + run: | + set -euo pipefail + shopt -s nullglob + + mkdir -p release-evidence + cp sbom-sidecars/*/*.spdx.json release-evidence/ + cp sbom-sidecars/*/*.licenses.json release-evidence/ + cp sbom-sidecars/*/*.licenses.md release-evidence/ + cp sbom-sidecars/*/*.build-evidence.json release-evidence/ + + uv run --project scripts/sbom --python 3.12 \ + isaaccapture-sbom merge-manifests \ + --output release-evidence/isaaccapture-sbom-manifest.json \ + sbom-sidecars/*/manifest.json + + # Rehearses the published verification instructions against the exact + # bytes a consumer will download, and re-checks every advertised digest. + uv run --project scripts/sbom --python 3.12 \ + isaaccapture-sbom check-set \ + --manifest release-evidence/isaaccapture-sbom-manifest.json \ + --wheel-dir wheels --evidence-dir release-evidence + + ls -la release-evidence + + - name: Attach evidence to the release + env: + GH_TOKEN: ${{ github.token }} + TAG: ${{ github.ref_name }} + run: | + set -euo pipefail + + # release-tag.yml pushes the tag before it creates the draft release, so + # this job can start first. Wait for the release rather than creating one, + # which would bypass the draft/approval step. + deadline=$(( SECONDS + 900 )) + until gh release view "${TAG}" >/dev/null 2>&1; do + if (( SECONDS >= deadline )); then + echo "::error::No release exists for ${TAG} after 15 minutes; evidence not attached." + exit 1 + fi + echo "Waiting for the ${TAG} release to be created..." + sleep 20 + done + + # --clobber makes a retry after a partial upload idempotent. Draft state + # and release notes are left exactly as they were. + gh release upload "${TAG}" release-evidence/* --clobber + + echo "Attached $(ls release-evidence | wc -l) evidence file(s) to ${TAG}" + publish-wheel: # to internal Artifactory runs-on: [self-hosted, linux] permissions: {} @@ -1067,6 +1237,69 @@ jobs: merge-multiple: true path: wheels + - name: Download wheel SBOM sidecars + uses: actions/download-artifact@v7 + with: + pattern: isaaccapture-sbom-* + path: sbom-sidecars + + # Nothing is published until every variant the matrix produces is here and + # still hashes to what its own manifest recorded. + - name: Verify wheel variant coverage and digests + run: | + set -euo pipefail + + # Mirrors the build-ubuntu Release matrix: 2 architectures x 3 Pythons. + platforms=(x86_64 aarch64) + abis=(cp311 cp312 cp313) + expected=$(( ${#platforms[@]} * ${#abis[@]} )) + + shopt -s nullglob + manifests=(sbom-sidecars/*/manifest.json) + if (( ${#manifests[@]} != expected )); then + echo "Expected ${expected} SBOM manifests, found ${#manifests[@]}" + ls -la sbom-sidecars || true + exit 1 + fi + + published=() + for manifest in "${manifests[@]}"; do + while IFS=$'\t' read -r filename digest; do + wheel="wheels/${filename}" + if [[ ! -f "${wheel}" ]]; then + echo "Manifest names ${filename}, which is not among the downloaded wheels" + exit 1 + fi + actual="$(sha256sum "${wheel}" | cut -d' ' -f1)" + if [[ "${actual}" != "${digest}" ]]; then + echo "${filename} was modified after its digest was bound" + echo " manifest: ${digest}" + echo " actual: ${actual}" + exit 1 + fi + published+=("${filename}") + done < <(jq -r '.wheels[] | [.filename, .sha256] | @tsv' "${manifest}") + done + + for platform in "${platforms[@]}"; do + for abi in "${abis[@]}"; do + found=0 + for filename in "${published[@]}"; do + if [[ "${filename}" == *"-${abi}-${abi}-manylinux"*"_${platform}.whl" ]]; then + found=1 + break + fi + done + if (( found == 0 )); then + echo "No wheel was published for ${abi} on ${platform}" + printf ' have: %s\n' "${published[@]}" + exit 1 + fi + done + done + + echo "All ${expected} wheel variants present and matching their recorded digests" + - name: Upload wheel(s) to Artifactory id: upload-artifactory env: diff --git a/.gitignore b/.gitignore index f939df2136..ae5e75dbdf 100644 --- a/.gitignore +++ b/.gitignore @@ -84,3 +84,5 @@ tmp/ !/examples/mujoco_xr/python/isaacteleop_examples/mujoco_xr/assets/leader/leader_gripper.xml /examples/mujoco_xr/python/isaacteleop_examples/mujoco_xr/assets/follower/* !/examples/mujoco_xr/python/isaacteleop_examples/mujoco_xr/assets/follower/follower_arm.xml +# Wheel SBOM and license evidence written by the collector in scripts/sbom/ +/sbom/ diff --git a/AGENTS.md b/AGENTS.md index b75eea35c6..2e28c6cfeb 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -236,6 +236,29 @@ pre-commit install --hook-type commit-msg - If a hook failure shows **missing or non-obvious repo policy** (not a one-off typo), you **must** add a **short** reminder under **Mandatory learning loop** rules to the right `AGENTS.md` or adjacent **`//` comments** so the next run does not repeat it—unless it is already documented. +## The wheel SBOM + +Published wheels carry a contents SBOM and license evidence derived from the +build itself. There is **no checked-in dependency registry** and adding one would +be a second source of truth — declare a dependency the way you already would and +it appears in the next wheel's inventory. + +Two things fail the Release build rather than shipping a gap, and neither is +fixed by excluding a file or writing down an assumption: + +- **a wheel member the build cannot explain** — teach the collector that route; +- **a redistributed component with no license text** — obtain the terms. + +The collector is a project of its own: `uv run --project scripts/sbom +isaaccapture-sbom ...`. Its dependency pins live in `scripts/sbom/pyproject.toml` +and nowhere else — never restate them as `--with` flags in a workflow or a doc. + +**Read [`docs/source/references/sbom.rst`](docs/source/references/sbom.rst) +first** if you are changing anything under `scripts/sbom/` or +`tests/python/sbom/`, changing how a dependency reaches a wheel, or touching the +wheel-packaging steps in `build-ubuntu.yml`. It covers discovery, license +matching and the maintenance rules; nothing here repeats them. + ## Mandatory learning loop (AGENTS.md and comments) **Hard requirement:** When **any** of the following happens, you **must** complete steps 1–3 **before** you end the session or move on as if the work were complete: diff --git a/CMakeLists.txt b/CMakeLists.txt index e08859aff5..61fe9b98d5 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,7 +1,16 @@ # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -cmake_minimum_required(VERSION 3.24...3.25) +# 3.27 for cmake_file_api(): the SBOM collector reads this build's link and +# compile graph from the file API reply, and that command is what lets the +# query live here rather than in whatever invoked cmake. No range: raising the +# minimum already opts into every policy through 3.27, and nothing here needs +# the ones after it. +cmake_minimum_required(VERSION 3.27) + +# Ask for the codemodel this configure run. Generator-independent, and produced +# by the same run that writes it -- see docs: references/sbom. +cmake_file_api(QUERY API_VERSION 1 CODEMODEL 2) # Workaround for older CMake policy compatibility in dependencies set(CMAKE_POLICY_VERSION_MINIMUM 3.5 CACHE STRING "Minimum CMake policy version") diff --git a/deps/README.md b/deps/README.md index b4b08ce8ab..7bba06acb3 100644 --- a/deps/README.md +++ b/deps/README.md @@ -112,7 +112,7 @@ machine and are located via `find_package`. - **Version**: 31ba1a50e5397e00a304dbadc76531740e89ee48 (v3.29.0) - **Purpose**: reference license texts. The SBOM collector names a dependency's license by matching the text that dependency ships against these; nothing from - this checkout is redistributed. + this checkout is redistributed. See `docs/source/references/sbom.rst`. - **Build**: data only, no build step. - **License**: CC0-1.0 diff --git a/docs/source/index.rst b/docs/source/index.rst index 31cda57dd8..d606c49786 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -78,6 +78,7 @@ Table of Contents references/requirements references/build + references/sbom references/generated_trackers references/retargeting/index references/device_provider_monitoring diff --git a/docs/source/references/sbom.rst b/docs/source/references/sbom.rst new file mode 100644 index 0000000000..328cf05b23 --- /dev/null +++ b/docs/source/references/sbom.rst @@ -0,0 +1,255 @@ +.. SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +.. SPDX-License-Identifier: Apache-2.0 + +Wheel contents SBOM and license evidence +======================================== + +Every published ``isaaccapture`` wheel carries a machine-readable inventory of what it +redistributes, plus the license text behind each obligation. You can inspect both +without installing the package, running it, or contacting a hosted service. + +What a wheel carries +-------------------- + +.. code-block:: text + + isaaccapture---.whl + └── isaaccapture-.dist-info/ + ├── METADATA # License-Expression + one License-File per text below + ├── licenses/ + │ ├── LICENSE.md # this project's own license + │ ├── LICENSES/*.txt # texts the SPDX headers on this project's sources name + │ ├── THIRD-PARTY-NOTICES.md # component -> license -> text, with roles + │ └── third-party// # each redistributed component's own file, verbatim + └── sboms/ + └── .spdx.json # SPDX 2.3 contents inventory + +Every text there comes from this repository or from a component the wheel redistributes. +The SPDX reference texts the collector matches against are a separate thing and are never +packaged — see `How licenses are determined`_. + +``.dist-info/sboms/`` holds the contents SBOM and nothing else, which is what the +`binary distribution format `_ +reserves it for. Reports and manifests are published beside the wheel instead. + +Verifying a wheel +----------------- + +Integrity and provenance are the packaging ecosystem's job, not this document's: + +``RECORD`` + Every wheel already carries a ``sha256`` and size for each of its files, and + installers check them. That is the integrity mechanism for a wheel's contents. + +`PEP 740 `_ attestations + Signed attestations binding a publisher identity to an artifact's hash, served + through PyPI's Integrity API. That is the provenance mechanism. + +The SBOM *describes* what those files are — where each came from, under what license. +Its per-file digests are there so the description can be shown to belong to the wheel in +front of you, and so it can be checked against ``RECORD``; they are not a second root of +trust. + +.. code-block:: bash + + git clone --depth 1 https://github.com/NVIDIA/IsaacCapture + uv run --project IsaacCapture/scripts/sbom isaaccapture-sbom check \ + --wheel isaaccapture---.whl + +The collector is a project of its own, :code-file:`scripts/sbom/pyproject.toml`, so its +dependencies come with it and are pinned there rather than at each call site. They are the +packages that own the formats involved — ``installer`` for ``RECORD``, ``packaging`` for +core metadata, ``license-expression`` for SPDX expressions, ``packageurl-python`` for +purls, ``pyelftools`` for ELF. + +One command, widening with whatever you have. With just a wheel it confirms that + +- the SBOM covers every file in the wheel except ``RECORD`` and the SBOM itself, +- each digest the SBOM records matches the bytes present, and agrees with ``RECORD``, +- ``RECORD`` itself agrees with the archive, so the wheel is still installable, +- the packaged notices are there, and every ``LicenseRef-`` has its text embedded. + +Add ``--build-evidence`` and it also checks that the build accounts for every member; +add ``--manifest`` and it checks the wheel against the digest bound at publication and +the sidecars beside it. ``check-set`` runs the whole thing over every wheel in a release. + +To read the inventory directly, unzip ``.dist-info/sboms/*.spdx.json`` and feed it to any +SPDX 2.3 consumer. CI validates each published document with ``spdx-tools`` before release. + +Evidence published beside a wheel +--------------------------------- + +Tagged releases attach the following to the GitHub release, so the evidence outlives the +CI artifacts it travelled in: + +``.spdx.json`` + Byte-identical copy of the document embedded in that wheel. + +``.licenses.md`` / ``.licenses.json`` + Component-to-license report: every redistributed component, the SPDX expression + concluded for it, the file that expression was read from, how the component reaches + the wheel, what is supplied by the installation environment instead, and what is + excluded and why. + +``.build-evidence.json`` + Resolved commits, declared constraints, archive digests and build options recorded on + the machine that built the wheel. + +``isaaccapture-sbom-manifest.json`` + Maps each published wheel filename and SHA-256 to its evidence files and their digests. + The wheel digest is bound here, after the last change to the wheel, so a wheel that was + touched again no longer matches. + +Downloading a whole release? Check the wheels and the evidence against the manifest in +one step: + +.. code-block:: bash + + uv run --project IsaacCapture/scripts/sbom isaaccapture-sbom check-set \ + --manifest isaaccapture-sbom-manifest.json \ + --wheel-dir . --evidence-dir . + +What is in scope +---------------- + +The document describes **the contents of one built wheel**: compiled-in static and +header-only dependencies, bundled shared libraries, vendored sources, SDK binaries and +packaged assets, for all six published variants (``x86_64`` and ``aarch64`` × CPython +3.11, 3.12, 3.13). + +Deliberately **not** described by it: + +- Build-time-only inputs — test frameworks, code generators, CMake modules, toolchains. + They appear in the report under *Fetched for this build but not in this wheel*; a full + build inventory is separate work. +- Source distributions and the ``pip install .`` path, which produce different contents. +- Containers and the web client, which are distributed on their own. +- Dependencies resolved at install time. The wheel's ``Requires-Dist`` metadata stays + authoritative for those; the document records the requirement, never a license for it. + +How the inventory is worked out +------------------------------- + +There is no checked-in dependency list. Every entry is derived from the build that +produced the wheel, so a dependency that builds is a dependency that appears: + +``build/_deps/*-src`` + FetchContent checkouts. ``git`` in each one supplies the commit, the remote, and from + the remote the supplier, homepage and purl. + +CMake's file API codemodel + The build graph, as CMake itself describes it: which sources were compiled into which + binary, which targets it depends on, and — through include paths — which header-only + dependencies it was compiled against. Followed transitively, so a library linked into a + library linked into the wheel is still reported. The top-level ``CMakeLists.txt`` asks + for it with ``cmake_file_api()``, which is why the project requires CMake 3.27; reading + the generator's own output instead would only work under Makefiles, since Ninja writes + no ``link.txt``. + +Archives in the repository + Every member of every tarball is hashed. A wheel member that matches one is recorded as + having been unpacked from that archive. + +The working tree and the build tree + What remains is matched against the project's own sources and its generated output. + +**Every member of the wheel must be explained by one of those.** A file that is not stops +the build rather than being quietly omitted — that is the check that keeps the inventory +honest as the build changes. + +Post-processing is recorded as such. ``patchelf``, ``auditwheel``'s RPATH rewrite and the +build's MJCF asset stripping all leave bytes that no longer hash to their origin, so those +members are reported as *derived from* a named input, with the original digest, rather +than as an exact copy. + +How licenses are determined +--------------------------- + +Licenses are never inferred from this project's license, and never read from a list +someone maintained by hand. For each component the collector finds the license and notice +files the component itself ships — ``LICENSE``, ``COPYING``, ``BSD-LICENSE``, a ``LICENSES/`` +pool, or, where upstream ships no file at all, the license section of its README — and +matches each text against a corpus of SPDX reference texts. + +Those reference texts are a dependency like any other. :code-file:`deps/third_party/CMakeLists.txt` +fetches +``spdx/license-list-data`` at a pinned commit into ``_deps/license-list-data-src``, the same +mechanism and the same place as every other dependency — which means the collector +discovers it as a component of the build and reports it, alongside the rest, as fetched but +shipping nothing. Pinning is by commit, so no tag or branch upstream can move underneath a +build, and each report records the commit its identifications were made against. + +Consulting the list is not shipping it. Every current identifier is available to match +against, so a dependency arriving under any of them lands with a real SPDX id. What the +wheel carries for a third-party component is that component's own license file, verbatim, +copyright holders and all; a reference text is never substituted for it and no file from +the checkout is packaged. The canonical texts that do ship under +``.dist-info/licenses/LICENSES/`` are this repository's own REUSE pool — the texts the +SPDX headers on its sources name — and have nothing to do with the matching. + +A match needs a canonical license text to be almost entirely present *and* to account for a +real share of the file, which is what stops a license quoted in the appendix of a long +agreement from being mistaken for that agreement's own terms. A file that genuinely carries +two licenses is reported as carrying both. + +A text that matches nothing is not guessed at. It becomes a ``LicenseRef-`` whose full text +travels in the document, which says "these are the terms that shipped" without claiming to +know which license they are. Those components are listed in the report under *License +evidence gaps*. + +A component that ships **no** license text at all blocks publication, because packaging the +text is the obligation the wheel has to meet. + +A shared library that none of those indexes explains did not come from this build, so the +last thing asked is the build machine itself: resolve the library's SONAME, and take its +license from the supplying package's copyright file, or from a license file beside the +library for toolchains that install outside the package manager. Resolving the SONAME is +what licenses the claim — a library the machine cannot find is reported as unexplained, +not attributed to something nobody located. No repair tool's directory name is consulted, +so a library the build produced is attributed to the build wherever it ends up. + +Running the collector locally +----------------------------- + +Against a configured build tree: + +.. code-block:: bash + + cmake -B build -DCMAKE_BUILD_TYPE=Release + cmake --build build + cmake --install build + + # Configuring fetched the SPDX reference texts into + # build/_deps/license-list-data-src/json, which is where `build` looks + # unless --license-data says otherwise. + uv run --project scripts/sbom isaaccapture-sbom build \ + --build-dir build --wheel install/wheels/.whl --out-dir sbom + uv run --project scripts/sbom isaaccapture-sbom check --wheel install/wheels/.whl \ + --manifest sbom/manifest.json --build-evidence sbom/.build-evidence.json + +``build`` rewrites the wheel in place, so run it on a repaired wheel and do not touch the +wheel afterwards. Set ``SOURCE_DATE_EPOCH`` to make the result reproducible. + +``--project`` points ``uv`` at the collector, not at this repository: it resolves the +collector's own pins and runs it from wherever you are. The checkout it inventories is the +one you are standing in, or ``--repo-root``. + +Maintaining it +-------------- + +Adding a dependency takes no SBOM change: declare it the way you already would — a +``FetchContent_Declare``, a tarball under ``deps/`` — and it appears in the next wheel's +inventory with its identity, its license and the route it took to get there. + +Two things do need attention: + +- **A component that ships no license text stops the build.** The fix is to obtain the + terms, not to record an assumption. +- **A new member that the build cannot explain stops the build.** That means something + reached the wheel by a route the collector cannot see; teach it that route rather than + excluding the file. + +Moving to a newer SPDX license list is a dependency bump like any other: change +``SPDX_LICENSE_DATA_COMMIT`` in :code-file:`deps/third_party/CMakeLists.txt`, with its release +comment. There is no separate list of identifiers to extend — the whole SPDX list comes +with the checkout. diff --git a/pyproject.toml b/pyproject.toml index 3a234232fe..b220d0664f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -15,6 +15,7 @@ description = "Isaac Teleop" readme = "README.md" requires-python = ">=3.11" license = "Apache-2.0" +license-files = ["LICENSE.md", "LICENSES/*.txt"] authors = [ { name = "NVIDIA" }, ] @@ -100,11 +101,11 @@ package = false isaacteleop = { path = "src/compat" } [tool.scikit-build] -# Floor matches CMakeLists' cmake_minimum_required (3.24, for `cmake --fresh`). +# Floor matches CMakeLists' cmake_minimum_required (3.27, for cmake_file_api). # Ubuntu 22.04's apt CMake is 3.22 and no longer satisfies it, so scikit-build-core # fetches CMake from PyPI there; the <4 cap keeps that fetch on the 3.x line rather # than silently jumping to 4.x. -cmake.version = ">=3.24,<4" +cmake.version = ">=3.27,<4" cmake.build-type = "Release" # Keep the CMake build tree under build-wheel// (e.g. diff --git a/scripts/sbom/pyproject.toml b/scripts/sbom/pyproject.toml new file mode 100644 index 0000000000..2136820165 --- /dev/null +++ b/scripts/sbom/pyproject.toml @@ -0,0 +1,41 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# The wheel SBOM collector. A project of its own so its dependencies are +# declared once, here, rather than repeated at every call site. +# +# uv run --project scripts/sbom isaaccapture-sbom --help +# +# See docs: references/sbom. + +[project] +name = "isaaccapture-sbom" +version = "1.0.0" +description = "Discover and package the contents SBOM for an Isaac Capture wheel" +requires-python = ">=3.11" +license = "Apache-2.0" +authors = [{ name = "NVIDIA" }] + +# Each owns a format the collector would otherwise parse by hand. Pinned: the +# documents they produce are published, so an upgrade is a deliberate change. +dependencies = [ + "installer==1.0.1", # RECORD + "license-expression==30.4.4", # SPDX expression grammar + "packageurl-python==0.17.6", # purls + "packaging==26.3", # core metadata, requirements + "pyelftools==0.33", # ELF dynamic section (auditwheel uses it too) +] + +[project.optional-dependencies] +# Third-party conformance check for the documents this emits. +dev = ["pytest", "spdx-tools==0.8.3"] + +[project.scripts] +isaaccapture-sbom = "sbom.cli:main" + +[build-system] +requires = ["setuptools>=77"] +build-backend = "setuptools.build_meta" + +[tool.setuptools.packages.find] +include = ["sbom*"] diff --git a/scripts/sbom/sbom/__init__.py b/scripts/sbom/sbom/__init__.py new file mode 100644 index 0000000000..073da5fe74 --- /dev/null +++ b/scripts/sbom/sbom/__init__.py @@ -0,0 +1,9 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Wheel contents SBOM and license evidence collector.""" + +TOOL_NAME = "isaaccapture-sbom" +# Bump on any change that alters the shape of an emitted document or report. +TOOL_VERSION = "1.0.0" +SPDX_VERSION = "SPDX-2.3" diff --git a/scripts/sbom/sbom/build.py b/scripts/sbom/sbom/build.py new file mode 100644 index 0000000000..ae73dbc681 --- /dev/null +++ b/scripts/sbom/sbom/build.py @@ -0,0 +1,265 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Turn a repaired wheel plus its build tree into a wheel that carries both.""" + +from __future__ import annotations + +import hashlib +import json +import os +import shutil +import tempfile +from dataclasses import replace +from datetime import datetime, timezone +from pathlib import Path + +from . import TOOL_NAME, TOOL_VERSION +from . import document as document_module +from . import evidence as evidence_module +from . import licensing +from . import inventory as inventory_module +from . import report as report_module +from . import wheelfile +from .inventory import Attribution +from .wheelfile import Entry + +PROJECT = { + "supplier": "Organization: NVIDIA CORPORATION", + "homepage": "https://github.com/NVIDIA/IsaacCapture", + "license": "Apache-2.0", + "document_namespace": "https://github.com/NVIDIA/IsaacCapture/spdx", +} + + +class BuildError(Exception): + """The wheel cannot be published with the evidence available.""" + + +def _entry(name: str, data: bytes) -> Entry: + return Entry( + name=name, + size=len(data), + sha256=hashlib.sha256(data).hexdigest(), + sha1=hashlib.sha1(data).hexdigest(), # noqa: S324 - SPDX 2.3 requires SHA1 + ) + + +def _declared_path(dist_info: str, member: str) -> str: + """What METADATA calls a packaged license file. + + PEP 639 states License-File relative to `.dist-info/licenses/`, which is + what setuptools already does for the project's own texts; declaring the + member path instead yields an entry that resolves to nothing. + """ + return member.removeprefix(f"{dist_info}/licenses/") + + +def sha256_file(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1 << 20), b""): + digest.update(chunk) + return digest.hexdigest() + + +def default_license_data(build_dir: Path) -> Path: + """Where deps/third_party materializes the SPDX license list for this build.""" + return build_dir / "_deps" / "license-list-data-src" / "json" + + +def build( + repo_root: Path, + build_dir: Path, + wheel_path: Path, + out_dir: Path, + license_data: Path | None = None, + system_resolver=None, +) -> dict: + """Package evidence into `wheel_path` in place and write the sidecars.""" + licensing.load_corpus(license_data or default_license_data(build_dir)) + discovery = evidence_module.discover(repo_root, build_dir) + wheel = wheelfile.scan(wheel_path, analyze_elf=True) + already = [ + item.name for item in wheel.entries if f"{wheel.dist_info}/sboms/" in item.name + ] + if already: + raise BuildError( + f"{wheel_path.name} already carries {already[0]}. Run this on a freshly " + "repaired wheel; rewriting one twice would duplicate its members." + ) + metadata_raw, _ = wheelfile.read_metadata(wheel_path, wheel.dist_info) + + resolver = inventory_module.Resolver(discovery, system_resolver) + inventory = inventory_module.build( + wheel, metadata_raw, resolver, discovery.staged_root + ) + + if inventory.unattributed: + raise BuildError( + "these wheel members could not be traced to anything this build " + "produced or fetched:\n " + + "\n ".join(inventory.unattributed) + + "\nThey reached the wheel by a route the collector cannot see." + ) + + # A REUSE LICENSES/ pool is a real text even though it names no single + # expression for the component, so it satisfies the packaging obligation. + missing = [ + key + for key, component in inventory.components_present.items() + if not any(item.kind in ("grant", "pool") for item in component.evidence) + ] + if missing: + raise BuildError( + "no license text was found in the files of redistributed component(s): " + + ", ".join(missing) + + "\nThe component ships no LICENSE/COPYING file and states no grant in " + "its README; obtain the terms before publishing it." + ) + + # Everything the wheel gains, computed before a byte is written so the + # document can describe the wheel it is about to be embedded in. + additions: dict[str, bytes] = {} + packaged_paths: list[str] = [] + for key, component in inventory.components_present.items(): + for item in component.evidence: + target = report_module.packaged_license_path(wheel.dist_info, key, item) + if target in additions: + continue + additions[target] = item.text.encode("utf-8") + packaged_paths.append(_declared_path(wheel.dist_info, target)) + + notices_path = f"{wheel.dist_info}/licenses/THIRD-PARTY-NOTICES.md" + additions[notices_path] = report_module.notices_markdown( + inventory, wheel.dist_info, wheel_path.name + ).encode("utf-8") + packaged_paths.append(_declared_path(wheel.dist_info, notices_path)) + + patched_metadata = wheelfile.add_license_files(metadata_raw, sorted(packaged_paths)) + metadata_declared = patched_metadata != metadata_raw + replacements = {f"{wheel.dist_info}/METADATA": patched_metadata} + + sbom_name = f"{wheel.dist_info}/sboms/{wheel_path.name}.spdx.json" + excluded = [f"{wheel.dist_info}/RECORD", sbom_name] + + projected = [item for item in wheel.entries if item.name not in replacements] + [ + _entry(name, data) for name, data in {**replacements, **additions}.items() + ] + projected_wheel = replace(wheel, entries=tuple(projected)) + for name in [*additions, *replacements]: + inventory.attributions[name] = Attribution( + path=name, + origin="metadata", + detail="license evidence packaged by the SBOM collector" + if name in additions + else "distribution metadata, updated to declare the packaged license files", + ) + # The document is added below, after it has been rendered; record it now so + # the evidence accounts for every member of the finished wheel. + inventory.attributions[sbom_name] = Attribution( + path=sbom_name, + origin="metadata", + detail="the contents SBOM this collector embedded", + ) + + evidence_doc = evidence_module.document(discovery, wheel_path.name, inventory) + evidence_doc["ci"] = environment_summary() + evidence_doc["system_libraries"] = resolver.system_libraries + + spdx = document_module.build_document( + PROJECT, evidence_doc, projected_wheel, inventory, excluded + ) + spdx_bytes = json.dumps(spdx, indent=2, sort_keys=True).encode("utf-8") + b"\n" + additions[sbom_name] = spdx_bytes + + with tempfile.TemporaryDirectory(dir=str(wheel_path.parent)) as scratch: + staged = Path(scratch) / wheel_path.name + wheelfile.rewrite( + wheel_path, + staged, + additions=additions, + replacements=replacements, + dist_info=wheel.dist_info, + ) + shutil.move(str(staged), str(wheel_path)) + + payload = report_module.report(inventory, evidence_doc, wheel_path.name) + payload["packaged_license_files"] = sorted(packaged_paths) + payload["metadata_license_files_declared"] = metadata_declared + + out_dir.mkdir(parents=True, exist_ok=True) + stem = wheel_path.name.removesuffix(".whl") + sidecars = { + "spdx": out_dir / f"{stem}.spdx.json", + "license_report_json": out_dir / f"{stem}.licenses.json", + "license_report_md": out_dir / f"{stem}.licenses.md", + "build_evidence": out_dir / f"{stem}.build-evidence.json", + } + sidecars["spdx"].write_bytes(spdx_bytes) + sidecars["license_report_json"].write_text( + json.dumps(payload, indent=2, sort_keys=True) + "\n", encoding="utf-8" + ) + sidecars["license_report_md"].write_text( + report_module.report_markdown(payload), encoding="utf-8" + ) + sidecars["build_evidence"].write_text( + json.dumps(evidence_doc, indent=2, sort_keys=True) + "\n", encoding="utf-8" + ) + + # The wheel digest is bound here, after the last mutation: a wheel that is + # touched again no longer matches its own manifest entry. + return { + "schema": "isaaccapture-sbom-manifest/1", + "generated_at": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"), + "tool": {"name": TOOL_NAME, "version": TOOL_VERSION}, + "wheels": [ + { + "filename": wheel_path.name, + "sha256": sha256_file(wheel_path), + "size": wheel_path.stat().st_size, + "sbom_in_wheel": sbom_name, + "components_without_license_evidence": payload[ + "components_without_license_evidence" + ], + "components_with_unidentified_license": payload[ + "components_with_unidentified_license" + ], + "sidecars": { + kind: { + "filename": path.name, + "sha256": sha256_file(path), + "size": path.stat().st_size, + } + for kind, path in sidecars.items() + }, + } + ], + } + + +def merge_manifests(paths: list[Path]) -> dict: + """Fold per-variant manifests into the one a release advertises.""" + wheels: list[dict] = [] + for path in sorted(paths): + wheels.extend(json.loads(path.read_text(encoding="utf-8"))["wheels"]) + names = [item["filename"] for item in wheels] + duplicates = sorted({name for name in names if names.count(name) > 1}) + if duplicates: + raise BuildError(f"the same wheel filename was published twice: {duplicates}") + return { + "schema": "isaaccapture-sbom-manifest/1", + "generated_at": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"), + "tool": {"name": TOOL_NAME, "version": TOOL_VERSION}, + "wheels": sorted(wheels, key=lambda item: item["filename"]), + } + + +def environment_summary() -> dict: + """Recorded so a rerun can tell a cold build from a cache-restored one.""" + return { + "runner_os": os.environ.get("RUNNER_OS"), + "github_run_id": os.environ.get("GITHUB_RUN_ID"), + "github_sha": os.environ.get("GITHUB_SHA"), + "github_ref": os.environ.get("GITHUB_REF"), + } diff --git a/scripts/sbom/sbom/cli.py b/scripts/sbom/sbom/cli.py new file mode 100644 index 0000000000..3ccf00e9c5 --- /dev/null +++ b/scripts/sbom/sbom/cli.py @@ -0,0 +1,187 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Command line for the wheel SBOM and license evidence collector.""" + +from __future__ import annotations + +import argparse +import json +import subprocess +import sys +from pathlib import Path + +from . import TOOL_NAME, TOOL_VERSION +from . import build as build_module +from . import evidence as evidence_module +from . import licensing +from . import validate as validate_module + + +def _repo_root(value: str | None) -> Path: + """The checkout to inventory: given, else the one the caller is standing in. + + Not derived from this file's location -- the collector is installed into an + environment of its own, so its own path says nothing about the checkout. + """ + if value: + return Path(value).resolve() + try: + result = subprocess.run( + ["git", "rev-parse", "--show-toplevel"], + capture_output=True, + text=True, + check=False, + timeout=30, + ) + except (OSError, subprocess.SubprocessError): + return Path.cwd() + return ( + Path(result.stdout.strip()).resolve() if result.returncode == 0 else Path.cwd() + ) + + +def _report(failures: list[str], subject: str) -> int: + if failures: + print(f"{subject}: {len(failures)} problem(s)", file=sys.stderr) + for failure in failures: + print(f" - {failure}", file=sys.stderr) + return 1 + print(f"{subject}: OK") + return 0 + + +def _cmd_build(args: argparse.Namespace) -> int: + repo_root = _repo_root(args.repo_root) + build_dir = Path(args.build_dir) + out_dir = Path(args.out_dir) + + license_data = ( + Path(args.license_data) + if args.license_data + else build_module.default_license_data(build_dir) + ) + + manifests: list[dict] = [] + for wheel in sorted(Path(item) for item in args.wheel): + manifests.append( + build_module.build(repo_root, build_dir, wheel, out_dir, license_data) + ) + print(f"packaged evidence into {wheel.name}") + + merged = { + **manifests[0], + "wheels": [item for manifest in manifests for item in manifest["wheels"]], + } + manifest_path = out_dir / "manifest.json" + manifest_path.write_text( + json.dumps(merged, indent=2, sort_keys=True) + "\n", encoding="utf-8" + ) + print(f"manifest: {manifest_path}") + + for item in merged["wheels"]: + for entry in item["components_with_unidentified_license"]: + print( + f"::warning::{item['filename']}: {entry['component']} ships a license " + "text that matches no SPDX reference" + ) + return 0 + + +def _cmd_check(args: argparse.Namespace) -> int: + status = 0 + for wheel in sorted(Path(item) for item in args.wheel): + failures = validate_module.check( + wheel, + Path(args.build_evidence) if args.build_evidence else None, + Path(args.manifest) if args.manifest else None, + Path(args.evidence_dir) if args.evidence_dir else None, + ) + status |= _report(failures, wheel.name) + return status + + +def _cmd_check_set(args: argparse.Namespace) -> int: + failures = validate_module.check_set( + Path(args.manifest), Path(args.wheel_dir), Path(args.evidence_dir) + ) + return _report(failures, Path(args.manifest).name) + + +def _cmd_merge_manifests(args: argparse.Namespace) -> int: + merged = build_module.merge_manifests([Path(item) for item in args.manifest]) + Path(args.output).write_text( + json.dumps(merged, indent=2, sort_keys=True) + "\n", encoding="utf-8" + ) + print(f"merged {len(merged['wheels'])} wheel record(s) -> {args.output}") + return 0 + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(prog=TOOL_NAME, description=__doc__) + parser.add_argument( + "--version", action="version", version=f"{TOOL_NAME} {TOOL_VERSION}" + ) + parser.add_argument( + "--repo-root", + help="IsaacCapture checkout (default: the git checkout you are standing in)", + ) + subparsers = parser.add_subparsers(dest="command", required=True) + + build = subparsers.add_parser( + "build", + help="discover what a build put in its wheels, then package the evidence into them", + ) + build.add_argument("--build-dir", required=True, help="configured CMake build tree") + build.add_argument("--wheel", required=True, action="append") + build.add_argument("--out-dir", required=True) + build.add_argument( + "--license-data", + help=( + "SPDX license list json/ directory fetched by deps/third_party " + "(default: /_deps/license-list-data-src/json)" + ), + ) + build.set_defaults(handler=_cmd_build) + + check = subparsers.add_parser( + "check", + help="check a wheel against the SBOM it carries, plus any evidence you supply", + ) + check.add_argument("--wheel", required=True, action="append") + check.add_argument( + "--build-evidence", help="also check that the build accounts for every member" + ) + check.add_argument( + "--manifest", help="also check the wheel's bound digest and sidecars" + ) + check.add_argument( + "--evidence-dir", help="where the sidecars live (default: beside the manifest)" + ) + check.set_defaults(handler=_cmd_check) + + check_set = subparsers.add_parser( + "check-set", help="run check over every wheel a merged manifest advertises" + ) + check_set.add_argument("--manifest", required=True) + check_set.add_argument("--wheel-dir", required=True) + check_set.add_argument("--evidence-dir", required=True) + check_set.set_defaults(handler=_cmd_check_set) + + merge = subparsers.add_parser( + "merge-manifests", help="fold per-variant manifests into one" + ) + merge.add_argument("--output", required=True) + merge.add_argument("manifest", nargs="+") + merge.set_defaults(handler=_cmd_merge_manifests) + + args = parser.parse_args(argv) + try: + return args.handler(args) + except ( + evidence_module.EvidenceError, + build_module.BuildError, + licensing.CorpusError, + ) as error: + print(f"error: {error}", file=sys.stderr) + return 2 diff --git a/scripts/sbom/sbom/discovery.py b/scripts/sbom/sbom/discovery.py new file mode 100644 index 0000000000..8a8ec62afe --- /dev/null +++ b/scripts/sbom/sbom/discovery.py @@ -0,0 +1,799 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Find out what went into a build by reading what the build left behind. + +There is no dependency list to keep in step with the code. A component exists +because a source tree was fetched into the build, and it reaches a wheel because +the link graph or a content hash says so. + +Four indexes do the work: + +* source trees ``_deps/*-src`` checkouts -> identity, from git +* build graph CMake's file API codemodel -> what links and compiles into what +* archives members of tarballs in the repo, by content hash +* files repo and build-tree files, by content hash +""" + +from __future__ import annotations + +import glob +import hashlib +import json +import re +import shutil +import subprocess +import tomllib +import tarfile +import zipfile +from dataclasses import dataclass +from pathlib import Path + +from packageurl import PackageURL + +from . import licensing + +# Directories that never hold an input worth indexing. +_SKIP_DIRS = { + ".git", + ".venv", + "venv", + "node_modules", + "__pycache__", + ".mypy_cache", + ".ruff_cache", +} +ARCHIVE_SUFFIXES = (".tar.gz", ".tgz", ".tar.xz", ".tar.bz2", ".tar", ".zip") +_MAX_ARCHIVE_BYTES = 1 << 31 + +_GITHUB_REMOTE = re.compile( + r"github\.com[:/](?P[^/]+)/(?P[^/]+?)(?:\.git)?/?$" +) +_GITCLONE_URL = re.compile(r'clone\b[^\n]*?"(?P[a-z]+[^"\s]*://[^"]+|git@[^"]+)"') +_GITCLONE_TAG = re.compile(r'(?[^"]+)"') +_INCLUDE_FLAG = re.compile(r"-(?:I|isystem)\s*(\S+)") +_DEPS_SOURCE = re.compile(r"_deps/(?P[A-Za-z0-9_.+-]+)-src(?:/|$)") + + +def sha256_bytes(data: bytes) -> str: + return hashlib.sha256(data).hexdigest() + + +def sha256_file(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1 << 20), b""): + digest.update(chunk) + return digest.hexdigest() + + +def _walk(root: Path): + stack = [root] + while stack: + current = stack.pop() + try: + entries = list(current.iterdir()) + except OSError: + continue + for entry in entries: + if entry.is_symlink(): + continue + if entry.is_dir(): + if entry.name not in _SKIP_DIRS: + stack.append(entry) + elif entry.is_file(): + yield entry + + +# ============================================================================== +# Source trees +# ============================================================================== + + +@dataclass(frozen=True) +class Component: + """Something a wheel can contain, identified from where it came from.""" + + key: str + name: str + kind: str # "source-tree" | "archive" | "system-library" + supplier: str + homepage: str + purl: str + version: str + download_location: str + source_info: str + root: Path | None = None + license_concluded: str = "NOASSERTION" + license_declared: str = "NOASSERTION" + evidence: tuple[licensing.LicenseEvidence, ...] = () + + +def git(repo: Path, *args: str) -> str | None: + """Run a read-only git query, or return None if git cannot answer.""" + try: + result = subprocess.run( + ["git", "-C", str(repo), *args], + capture_output=True, + text=True, + check=False, + timeout=60, + ) + except (OSError, subprocess.SubprocessError): + return None + return ( + result.stdout.strip() + if result.returncode == 0 and result.stdout.strip() + else None + ) + + +def _identity( + remote: str | None, name: str, commit: str | None +) -> tuple[str, str, str, str]: + """Supplier, homepage, purl and download location, read off the remote URL.""" + if remote: + match = _GITHUB_REMOTE.search(remote) + if match: + org, repo = match.group("org"), match.group("repo") + purl = PackageURL( + type="github", namespace=org, name=repo, version=commit + ).to_string() + download = f"git+https://github.com/{org}/{repo}.git" + ( + f"@{commit}" if commit else "" + ) + return ( + f"Organization: {org}", + f"https://github.com/{org}/{repo}", + purl, + download, + ) + purl = f"pkg:generic/{name}" + (f"@{commit}" if commit else "") + return ( + "NOASSERTION", + remote, + purl, + (f"git+{remote}@{commit}" if commit else remote), + ) + return "NOASSERTION", "NOASSERTION", f"pkg:generic/{name}", "NOASSERTION" + + +def _declared_constraint(deps_dir: Path, name: str) -> dict[str, str | None]: + clone = ( + deps_dir + / f"{name}-subbuild" + / f"{name}-populate-prefix" + / "tmp" + / f"{name}-populate-gitclone.cmake" + ) + if not clone.is_file(): + return {"url": None, "tag": None} + text = clone.read_text(encoding="utf-8", errors="replace") + url = _GITCLONE_URL.search(text) + tag = _GITCLONE_TAG.search(text) + return { + "url": url.group("url") if url else None, + "tag": tag.group("tag") if tag else None, + } + + +def discover_source_trees(deps_dir: Path) -> dict[str, Component]: + """Every FetchContent checkout under the build's ``_deps``.""" + components: dict[str, Component] = {} + for root in sorted(deps_dir.glob("*-src")): + if not root.is_dir(): + continue + key = root.name.removesuffix("-src") + declared = _declared_constraint(deps_dir, key) + commit = git(root, "rev-parse", "HEAD") + remote = git(root, "config", "--get", "remote.origin.url") or declared["url"] + describe = git(root, "describe", "--tags", "--always") + + supplier, homepage, purl, download = _identity(remote, key, commit) + evidence = licensing.discover_in_tree(key, root, root.name) + concluded, declared_expression = licensing.expression(evidence, key) + + details = [f"Fetched into {root.name}."] + if declared["tag"]: + details.append(f"Declared constraint: {declared['tag']}.") + if describe: + details.append(f"git describe: {describe}.") + + name_match = _GITHUB_REMOTE.search(remote) if remote else None + components[key] = Component( + key=key, + name=name_match.group("repo") if name_match else key, + kind="source-tree", + supplier=supplier, + homepage=homepage, + purl=purl, + version=commit or declared["tag"] or "NOASSERTION", + download_location=download, + source_info=" ".join(details), + root=root, + license_concluded=concluded, + license_declared=declared_expression, + evidence=tuple(evidence), + ) + return components + + +# ============================================================================== +# Build graph +# ============================================================================== + + +@dataclass(frozen=True) +class Artifact: + """One file a target produces, and the target that produced it.""" + + output: Path + target: str + target_id: str + + +@dataclass(frozen=True) +class _Node: + """A target as the CMake file API describes it.""" + + name: str + kind: str + sources: tuple[Path, ...] + includes: tuple[Path, ...] + dependencies: tuple[str, ...] + + +def _owner(path: Path, ownership: Ownership | None) -> str | None: + """The component a compiled source belongs to, if any.""" + match = _DEPS_SOURCE.search(path.as_posix()) + if match: + return match.group("name") + return ownership.owner_of(path) if ownership is not None else None + + +class FileApiError(Exception): + """The build tree carries no CMake file API reply to read.""" + + +class BuildGraph: + """The link and compile graph, read from CMake's own file API. + + The top-level ``CMakeLists.txt`` asks for the codemodel with + ``cmake_file_api()``, so any configure of this project leaves one behind, + under every generator. Scraping ``link.txt`` instead would only work for the + Makefile generators -- Ninja writes no such file. + """ + + def __init__(self, build_dir: Path, config: str | None = None) -> None: + self.build_dir = build_dir + self.artifacts: dict[Path, Artifact] = {} + self._nodes: dict[str, _Node] = {} + self._by_name: dict[str, list[Artifact]] = {} + self._load(config) + + def _load(self, config: str | None) -> None: + reply = self.build_dir / ".cmake" / "api" / "v1" / "reply" + models = sorted(reply.glob("codemodel-v2-*.json")) if reply.is_dir() else [] + if not models: + raise FileApiError( + f"no CMake file API codemodel under {reply}. Configure the project " + "with CMake 3.27 or newer, which is where cmake_file_api() asks for it." + ) + + codemodel = json.loads(models[-1].read_text(encoding="utf-8")) + source_root = Path(codemodel["paths"]["source"]) + build_root = Path(codemodel["paths"]["build"]) + + configurations = codemodel["configurations"] + chosen = next( + (item for item in configurations if item["name"] == (config or "")), + configurations[0], + ) + + def resolve(raw: str, root: Path) -> Path: + path = Path(raw) + return path if path.is_absolute() else (root / path) + + for entry in chosen["targets"]: + target = json.loads((reply / entry["jsonFile"]).read_text(encoding="utf-8")) + includes = { + resolve(item["path"], source_root).resolve() + for group in target.get("compileGroups", []) + for item in group.get("includes", []) + } + self._nodes[entry["id"]] = _Node( + name=target["name"], + kind=target["type"], + sources=tuple( + resolve(item["path"], source_root).resolve() + for item in target.get("sources", []) + ), + includes=tuple(sorted(includes)), + dependencies=tuple( + item["id"] for item in target.get("dependencies", []) + ), + ) + for item in target.get("artifacts", []): + output = resolve(item["path"], build_root).resolve() + artifact = Artifact( + output=output, target=target["name"], target_id=entry["id"] + ) + self.artifacts[output] = artifact + self._by_name.setdefault(output.name, []).append(artifact) + + def by_name(self, name: str) -> list[Artifact]: + return self._by_name.get(name, []) + + def _closure(self, target_id: str) -> list[_Node]: + """The target and everything it depends on, transitively.""" + seen: set[str] = set() + stack = [target_id] + found: list[_Node] = [] + while stack: + current = stack.pop() + if current in seen or current not in self._nodes: + continue + seen.add(current) + node = self._nodes[current] + found.append(node) + stack.extend(node.dependencies) + return found + + def contributions( + self, artifact: Artifact, ownership: Ownership | None = None + ) -> dict[str, set[str]]: + """Which components contributed to an artifact, and in what way. + + Walks the target's dependency closure. Compiled sources name a component + outright; include paths are the only trace a header-only dependency + leaves anywhere. + """ + found: dict[str, set[str]] = {} + + def note(key: str, how: str) -> None: + found.setdefault(key, set()).add(how) + + for node in self._closure(artifact.target_id): + for source in node.sources: + owner = _owner(source, ownership) + if owner: + note(owner, "compiled-source") + for include in node.includes: + match = _DEPS_SOURCE.search(include.as_posix()) + if match: + note(match.group("name"), "header-include") + elif ownership is not None: + # Which file in the directory was used is unknowable, so + # every component with files there is credited, the same + # way a fetched include directory is. + for owner in ownership.owners_under(include): + note(owner, "header-include") + return found + + def first_party( + self, artifact: Artifact, repo_root: Path, ownership: Ownership | None = None + ) -> bool: + """True when this build compiled the artifact from the project's own sources.""" + for node in self._closure(artifact.target_id): + if node.name != artifact.target: + continue + for source in node.sources: + if _owner(source, ownership): + continue + try: + source.relative_to(repo_root) + except ValueError: + continue + return True + return False + + +# ============================================================================== +# Ownership +# ============================================================================== + + +class Ownership: + """Which component a path belongs to. + + One lookup for every kind of third-party code, so recognising a new kind is + registering a root rather than adding a branch wherever paths are matched. + """ + + def __init__(self) -> None: + self._files: dict[Path, str] = {} + self._roots: list[tuple[Path, str]] = [] + + def register_root(self, root: Path, key: str) -> None: + self._roots.append((root.resolve(), key)) + + def register_file(self, path: Path, key: str) -> None: + self._files[path.resolve()] = key + + def owner_of(self, path: Path) -> str | None: + resolved = path.resolve() + owner = self._files.get(resolved) + if owner is not None: + return owner + # Longest root wins, so a vendored subdirectory beats its parent. + best: tuple[int, str] | None = None + for root, key in self._roots: + text = str(root) + if str(resolved) == text or str(resolved).startswith(text + "/"): + if best is None or len(text) > best[0]: + best = (len(text), key) + return best[1] if best else None + + def owners_under(self, directory: Path) -> set[str]: + """Components with files inside a directory, for include-path evidence.""" + prefix = str(directory.resolve()) + found = { + key + for path, key in self._files.items() + if str(path) == prefix or str(path).startswith(prefix + "/") + } + found.update( + key + for root, key in self._roots + if str(root) == prefix or str(root).startswith(prefix + "/") + ) + return found + + +# ============================================================================== +# Third-party sources vendored in this repository +# ============================================================================== +# Not everything third-party arrives through FetchContent. A file checked in +# here that someone else wrote says so in its REUSE header, which is the same +# thing the project states about its own files -- so the copyright holder is +# what separates them, not the directory it happens to sit in. + +# REUSE's tag where a file has one, and a plain copyright line where it does +# not -- upstream files usually carry the latter. +_COPYRIGHT_TAG = re.compile( + r"SPDX-FileCopyrightText:\s*(?P[^\n\r]+)" + r"|^[^\w\n]*Copyright\b(?P[^\n\r]+)", + re.MULTILINE, +) +_YEARS = re.compile(r"(copyright|\(c\)|©|\d{4}(\s*-\s*\d{4})?|,|\.)", re.IGNORECASE) + + +def project_authors(repo_root: Path) -> set[str]: + """Who the project says it is, read from its own packaging metadata.""" + pyproject = repo_root / "pyproject.toml" + if not pyproject.is_file(): + return set() + try: + data = tomllib.loads(pyproject.read_text(encoding="utf-8")) + except (OSError, tomllib.TOMLDecodeError): + return set() + return { + str(author["name"]).lower() + for author in data.get("project", {}).get("authors", []) + if author.get("name") + } + + +def _holder_of(path: Path) -> tuple[str, str | None] | None: + """The copyright holder and licence a file states about itself.""" + try: + head = path.open("rb").read(4096).decode("utf-8", "replace") + except OSError: + return None + match = _COPYRIGHT_TAG.search(head) + if not match: + return None + holder = _YEARS.sub(" ", match.group("holder") or match.group("plain") or "") + holder = " ".join(holder.split()).removesuffix(" All rights reserved").strip() + return (holder, licensing.read_spdx_tag(head)) if holder else None + + +def discover_vendored( + repo_root: Path, candidates: set[Path], authors: set[str] +) -> tuple[dict[str, Component], dict[str, list[Path]]]: + """Group checked-in files whose copyright holder is not the project's.""" + grouped: dict[str, list[Path]] = {} + licences: dict[str, set[str]] = {} + names: dict[str, str] = {} + + for path in sorted(candidates): + stated = _holder_of(path) + if stated is None: + continue + holder, expression = stated + if any(author in holder.lower() for author in authors): + continue + key = f"vendored:{licensing.spdx_safe(holder).lower()}" + grouped.setdefault(key, []).append(path) + names[key] = holder + if expression: + licences.setdefault(key, set()).add(expression) + + components: dict[str, Component] = {} + for key, paths in grouped.items(): + evidence = licensing.pool_evidence(repo_root, key, licences.get(key, set())) + concluded, declared = licensing.expression(evidence, key) + if licences.get(key): + declared = licensing.combine(sorted(licences[key])) + relative = ", ".join( + sorted(str(item.relative_to(repo_root)) for item in paths)[:4] + ) + components[key] = Component( + key=key, + name=names[key], + kind="vendored-source", + supplier=f"Organization: {names[key]}", + homepage="NOASSERTION", + purl=PackageURL( + type="generic", name=licensing.spdx_safe(names[key]).lower() + ).to_string(), + version="NOASSERTION", + download_location="NOASSERTION", + source_info=( + f"Checked into this repository rather than fetched; states its own " + f"copyright. Files: {relative}." + ), + license_concluded=concluded, + license_declared=declared, + evidence=tuple(evidence), + ) + return components, grouped + + +# ============================================================================== +# The build host +# ============================================================================== +# The last discovery source: a shared library that no index explains was not +# produced or fetched by this build, so ask the machine that supplied it. + +_SEARCH_ROOTS = ( + "/usr/lib/x86_64-linux-gnu", + "/usr/lib/aarch64-linux-gnu", + "/usr/lib", + "/lib/x86_64-linux-gnu", + "/lib/aarch64-linux-gnu", + "/usr/local/lib", +) + + +def resolve_system_library(soname: str) -> dict: + """Trace an auditwheel-vendored library back to the package that supplied it.""" + origin: dict = { + "soname": soname, + "resolved_path": None, + "package": None, + "copyright": None, + } + roots = [Path(item) for item in _SEARCH_ROOTS if Path(item).is_dir()] + roots += [Path(item) for item in sorted(glob.glob("/usr/local/cuda*/lib64"))] + + library = next((root / soname for root in roots if (root / soname).exists()), None) + if library is None: + return origin + library = library.resolve() + origin["resolved_path"] = str(library) + + if shutil.which("dpkg-query") is None: + return origin + try: + result = subprocess.run( + ["dpkg-query", "-S", str(library)], + capture_output=True, + text=True, + check=False, + timeout=60, + ) + except (OSError, subprocess.SubprocessError): + return origin + if result.returncode != 0 or ":" not in result.stdout: + return origin + + package = result.stdout.split(":", 1)[0].strip() + origin["package"] = package + copyright_file = Path("/usr/share/doc") / package / "copyright" + if copyright_file.is_file(): + origin["copyright"] = str(copyright_file) + return origin + + +def _license_near(library: Path) -> Path | None: + """A toolchain that installs outside the package manager still ships its terms. + + CUDA is the case that matters: the runtime lands in /usr/local/cuda*/lib64 with + EULA.txt a level or two up, and dpkg knows nothing about it. + """ + current = library.parent + for _ in range(3): + for entry in sorted(current.iterdir()) if current.is_dir() else []: + if entry.is_file() and licensing._classify(entry.name): # noqa: SLF001 + return entry + if entry.is_file() and entry.name.upper().startswith("EULA"): + return entry + if current.parent == current: + break + current = current.parent + return None + + +def system_component(soname: str, origin: dict) -> Component: + """A pre-built library the build machine supplied, named by what it came from.""" + evidence: list[licensing.LicenseEvidence] = [] + key = f"system:{soname}" + license_path = origin.get("copyright") + if not license_path and origin.get("resolved_path"): + found = _license_near(Path(origin["resolved_path"])) + license_path = str(found) if found else None + origin["license_file"] = license_path + if license_path and Path(license_path).is_file(): + text = Path(license_path).read_text(encoding="utf-8", errors="replace") + evidence.append( + licensing._evidence(key, "build-host", license_path, text, "grant") # noqa: SLF001 + ) + concluded, declared = licensing.expression(evidence, soname) + + package = origin.get("package") + details = [ + f"Vendored into the wheel by auditwheel from {origin.get('resolved_path') or soname}." + ] + if package: + details.append(f"Supplied by system package {package}.") + return Component( + key=key, + name=soname, + kind="system-library", + supplier=f"Organization: {package}" if package else "NOASSERTION", + homepage="NOASSERTION", + purl=PackageURL(type="generic", name=soname).to_string(), + version="NOASSERTION", + download_location="NOASSERTION", + source_info=" ".join(details), + license_concluded=concluded, + license_declared=declared, + evidence=tuple(evidence), + ) + + +# ============================================================================== +# Content indexes +# ============================================================================== + + +@dataclass(frozen=True) +class Member: + """A file inside an archive, or a file on disk.""" + + container: str + path: str + sha256: str + size: int + + +class ArchiveIndex: + """Members of every archive in the repo, so extracted bytes can be traced back.""" + + def __init__(self) -> None: + self.by_hash: dict[str, Member] = {} + self.by_name: dict[str, list[Member]] = {} + self.archives: dict[str, dict] = {} + + def add_archive(self, path: Path, display: str) -> None: + if path.stat().st_size > _MAX_ARCHIVE_BYTES: + return + members = self._read_members(path, display) + if members is None: + return + self.archives[display] = { + "path": display, + "sha256": sha256_file(path), + "size": path.stat().st_size, + "member_count": len(members), + } + for member, data in members: + self.by_hash.setdefault(member.sha256, member) + self.by_name.setdefault(Path(member.path).name, []).append(member) + self._maybe_license(display, member, data) + + def _maybe_license(self, display: str, member: Member, data: bytes) -> None: + name = Path(member.path).name + if Path(member.path).parent.as_posix() not in {".", ""}: + return # only the archive root states the bundle's own terms + if name.upper() == "VERSION" and len(data) < 256: + self.archives[display]["version_text"] = data.decode( + "utf-8", "replace" + ).strip() + return + if not licensing._classify(name): # noqa: SLF001 - same package + return + self.archives[display].setdefault("license_members", []).append((member, data)) + + @staticmethod + def _read_members(path: Path, display: str) -> list[tuple[Member, bytes]] | None: + """Hash every member; keep the bytes only of the few worth keeping. + + An SDK tarball expands to far more than it compresses to, so holding + every member would size peak memory by the largest archive rather than + by what is retained. + """ + collected: list[tuple[Member, bytes]] = [] + + def take(name: str, data: bytes) -> None: + member = Member(display, name, sha256_bytes(data), len(data)) + keep = b"" if Path(name).parent.as_posix() not in {".", ""} else data + collected.append((member, keep)) + + try: + if path.suffix == ".zip": + with zipfile.ZipFile(path) as archive: + for info in archive.infolist(): + if not info.is_dir(): + take(info.filename, archive.read(info.filename)) + else: + with tarfile.open(path) as archive: + for info in archive: + if not info.isfile(): + continue + handle = archive.extractfile(info) + if handle is not None: + take(info.name.removeprefix("./"), handle.read()) + except (OSError, tarfile.TarError, zipfile.BadZipFile): + return None + return collected + + +def discover_archives(repo_root: Path, skip: tuple[Path, ...] = ()) -> ArchiveIndex: + """Index archives held in the repository, not ones a build unpacked.""" + index = ArchiveIndex() + skipped = tuple(item.resolve() for item in skip) + for path in _walk(repo_root): + if any(str(path).startswith(str(item)) for item in skipped): + continue + name = path.name.lower() + if not name.endswith(ARCHIVE_SUFFIXES): + continue + try: + relative = path.relative_to(repo_root).as_posix() + except ValueError: + continue + index.add_archive(path, relative) + return index + + +class FileIndex: + """Files on disk by content hash, for tracing a copied wheel member back.""" + + def __init__(self) -> None: + self.by_hash: dict[str, str] = {} + self.by_suffix: dict[str, list[str]] = {} + self._digests: dict[str, str] = {} + + def digest_of(self, path: Path) -> str | None: + """The digest this index already computed for a path, if it indexed it.""" + return self._digests.get(str(path.resolve())) + + def add_tree( + self, root: Path, display_root: str, skip: tuple[Path, ...] = () + ) -> None: + skipped = tuple(item.resolve() for item in skip) + for path in _walk(root): + if any(str(path).startswith(str(item)) for item in skipped): + continue + try: + relative = path.relative_to(root).as_posix() + except ValueError: + continue + display = f"{display_root}/{relative}" + try: + digest = sha256_file(path) + except OSError: + continue + self._digests[str(path.resolve())] = digest + self.by_hash.setdefault(digest, display) + # Two trailing segments is enough to disambiguate an __init__.py. + self.by_suffix.setdefault( + "/".join(relative.rsplit("/", 2)[-2:]), [] + ).append(display) + + def path_suffix_match(self, wheel_path: str) -> str | None: + key = "/".join(wheel_path.rsplit("/", 2)[-2:]) + matches = self.by_suffix.get(key, []) + return matches[0] if len(matches) == 1 else None diff --git a/scripts/sbom/sbom/document.py b/scripts/sbom/sbom/document.py new file mode 100644 index 0000000000..2592e0e3c2 --- /dev/null +++ b/scripts/sbom/sbom/document.py @@ -0,0 +1,349 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Render the discovered inventory as an SPDX 2.3 JSON document.""" + +from __future__ import annotations + +import hashlib +import os +from datetime import datetime, timezone + +from packageurl import PackageURL +from packaging.requirements import Requirement + +from . import SPDX_VERSION, TOOL_NAME, TOOL_VERSION, licensing +from .discovery import Component +from .inventory import Inventory +from .wheelfile import WheelInfo + +NOASSERTION = "NOASSERTION" +# How a component reaches the wheel decides the relationship that says so. +_LINK_HOWS = {"compiled-source", "header-include", "prebuilt-library"} + + +def _spdx_id(prefix: str, value: str) -> str: + return f"SPDXRef-{prefix}-{licensing.spdx_safe(value)}" + + +def verification_code(sha1_digests: list[str]) -> str: + """SPDX package verification code: SHA1 over the sorted file SHA1s.""" + return hashlib.sha1("".join(sorted(sha1_digests)).encode("ascii")).hexdigest() # noqa: S324 + + +def _attribution_texts(component: Component) -> list[str]: + texts = [ + f"License evidence: {item.path} (sha256:{item.sha256}, {item.size} bytes, " + f"{item.origin}, {item.kind})" + + ( + " identified as " + + "; ".join( + f"{license_id} (containment {containment:.2f}, coverage {coverage:.2f})" + for license_id, containment, coverage in item.matches + ) + if item.matches + else "" + ) + for item in component.evidence + ] + if not any(item.kind == "grant" for item in component.evidence): + texts.append( + "No license grant was found in this component's own files; see the " + "component-to-license report." + ) + return texts + + +def _component_package(component: Component, roles: set[str]) -> dict: + package = { + "SPDXID": _spdx_id("Package", component.key), + "name": component.name, + "versionInfo": component.version, + "supplier": component.supplier, + "downloadLocation": component.download_location, + "filesAnalyzed": False, + "licenseConcluded": component.license_concluded, + "licenseDeclared": component.license_declared, + "copyrightText": NOASSERTION, + "externalRefs": [ + { + "referenceCategory": "PACKAGE-MANAGER", + "referenceType": "purl", + "referenceLocator": component.purl, + } + ], + "attributionTexts": _attribution_texts(component), + "sourceInfo": f"{component.source_info} Reaches this wheel as: {', '.join(sorted(roles))}.", + } + if component.homepage != NOASSERTION: + package["homepage"] = component.homepage + return package + + +def _file_license_refs(inventory: Inventory) -> set[str]: + """LicenseRef- identifiers that arrive through a file's own REUSE tag.""" + refs: set[str] = set() + for attribution in inventory.attributions.values(): + if attribution.spdx_tag: + refs.update(licensing.license_refs(attribution.spdx_tag)) + return refs + + +def _extracted_licenses(components: list[Component]) -> list[dict]: + """Texts for every LicenseRef the document uses, taken from the shipped file.""" + entries: dict[str, dict] = {} + for component in components: + refs = licensing.license_refs( + component.license_concluded, component.license_declared + ) + grants = [item for item in component.evidence if item.kind == "grant"] + if not refs or not grants: + continue + for ref in refs: + entries[ref] = { + "licenseId": ref, + "name": f"License terms shipped with {component.name}", + "extractedText": "\n\n".join(item.text for item in grants), + "comment": ( + "Matched no text in the SPDX reference corpus; reproduced here " + "verbatim from " + ", ".join(item.path for item in grants) + "." + ), + } + return [entries[key] for key in sorted(entries)] + + +def build_document( + project: dict, + evidence_doc: dict, + wheel: WheelInfo, + inventory: Inventory, + excluded_files: list[str], +) -> dict: + """Assemble the document. Callers validate it; this only renders.""" + files: list[dict] = [] + file_ids: dict[str, str] = {} + sha1_digests: list[str] = [] + + for index, entry in enumerate(sorted(wheel.entries, key=lambda item: item.name)): + if entry.name in excluded_files: + continue + file_id = _spdx_id("File", f"{index}-{entry.name.rsplit('/', 1)[-1]}") + file_ids[entry.name] = file_id + sha1_digests.append(entry.sha1) + + attribution = inventory.attributions.get(entry.name) + record = { + "SPDXID": file_id, + "fileName": f"./{entry.name}", + "checksums": [ + {"algorithm": "SHA1", "checksumValue": entry.sha1}, + {"algorithm": "SHA256", "checksumValue": entry.sha256}, + ], + "licenseConcluded": NOASSERTION, + "copyrightText": NOASSERTION, + } + if attribution and attribution.spdx_tag: + # A REUSE tag in the file is upstream stating that file's license. + record["licenseConcluded"] = attribution.spdx_tag + record["licenseInfoInFiles"] = [attribution.spdx_tag] + + comment = [] + if attribution: + comment.append(f"Origin: {attribution.origin}; {attribution.detail}") + if entry.is_elf: + comment.append( + f"ELF {entry.machine}; SONAME {entry.soname or 'none'}; " + f"DT_NEEDED: {', '.join(entry.needed) or 'none'}" + ) + if comment: + record["comment"] = " | ".join(comment) + files.append(record) + + code = verification_code(sha1_digests) + wheel_id = "SPDXRef-Package-wheel" + build = evidence_doc.get("build", {}) + + wheel_package = { + "SPDXID": wheel_id, + "name": wheel.distribution, + "versionInfo": wheel.version, + "packageFileName": wheel.path.name, + "supplier": project["supplier"], + "originator": project["supplier"], + "downloadLocation": NOASSERTION, + "filesAnalyzed": True, + "homepage": project["homepage"], + "licenseConcluded": project["license"], + "licenseDeclared": project["license"], + "licenseInfoFromFiles": [NOASSERTION], + "copyrightText": NOASSERTION, + "packageVerificationCode": { + "packageVerificationCodeValue": code, + "packageVerificationCodeExcludedFiles": [ + f"./{name}" for name in sorted(excluded_files) + ], + }, + "externalRefs": [ + { + "referenceCategory": "PACKAGE-MANAGER", + "referenceType": "purl", + "referenceLocator": PackageURL( + type="pypi", name=wheel.distribution, version=wheel.version + ).to_string(), + } + ], + "hasFiles": [file_ids[name] for name in sorted(file_ids)], + "sourceInfo": ( + f"Built from {project['homepage']} at commit " + f"{evidence_doc.get('project', {}).get('commit') or NOASSERTION}; " + f"arch {build.get('arch')}; CMAKE_BUILD_TYPE {build.get('config')}." + ), + } + + packages = [wheel_package] + relationships = [ + { + "spdxElementId": "SPDXRef-DOCUMENT", + "relatedSpdxElement": wheel_id, + "relationshipType": "DESCRIBES", + } + ] + + for key, component in inventory.components_present.items(): + roles = inventory.component_roles.get(key, set()) + package = _component_package(component, roles) + packages.append(package) + package_id = package["SPDXID"] + + # Linked in, copied in, or both; say whichever the evidence supports. + if roles & _LINK_HOWS: + relationships.append( + { + "spdxElementId": wheel_id, + "relatedSpdxElement": package_id, + "relationshipType": "STATIC_LINK", + } + ) + if roles - _LINK_HOWS: + relationships.append( + { + "spdxElementId": wheel_id, + "relatedSpdxElement": package_id, + "relationshipType": "CONTAINS", + } + ) + + for name in inventory.files_of(key): + if name not in file_ids: + continue + attribution = inventory.attributions[name] + exact = attribution.origin in {"copied", "archive-copy"} + relationships.append( + { + "spdxElementId": file_ids[name], + "relatedSpdxElement": package_id, + "relationshipType": "COPY_OF" if exact else "GENERATED_FROM", + } + ) + + for soname, consumers in inventory.external_runtime.items(): + package_id = _spdx_id("Package-external", soname) + packages.append( + { + "SPDXID": package_id, + "name": soname, + "versionInfo": NOASSERTION, + "supplier": NOASSERTION, + "downloadLocation": NOASSERTION, + "filesAnalyzed": False, + "licenseConcluded": NOASSERTION, + "licenseDeclared": NOASSERTION, + "copyrightText": NOASSERTION, + "comment": ( + "Supplied by the installation environment, not redistributed in " + f"this wheel. Required by: {', '.join(consumers)}." + ), + } + ) + relationships.append( + { + "spdxElementId": wheel_id, + "relatedSpdxElement": package_id, + "relationshipType": "DEPENDS_ON", + } + ) + + for requirement in inventory.requires_dist: + name = Requirement(requirement).name + package_id = _spdx_id("Package-pypi", requirement) + packages.append( + { + "SPDXID": package_id, + "name": name, + "versionInfo": NOASSERTION, + "supplier": NOASSERTION, + "downloadLocation": NOASSERTION, + "filesAnalyzed": False, + "licenseConcluded": NOASSERTION, + "licenseDeclared": NOASSERTION, + "copyrightText": NOASSERTION, + "externalRefs": [ + { + "referenceCategory": "PACKAGE-MANAGER", + "referenceType": "purl", + "referenceLocator": PackageURL( + type="pypi", name=name + ).to_string(), + } + ], + "comment": ( + f"Consumer requirement declared in wheel metadata: {requirement}. " + "Resolved at install time; this document states no license for it." + ), + } + ) + relationships.append( + { + "spdxElementId": wheel_id, + "relatedSpdxElement": package_id, + "relationshipType": "DEPENDS_ON", + } + ) + + # SOURCE_DATE_EPOCH makes the whole wheel reproducible, which is what lets a + # rebuild be compared against a published one byte for byte. + epoch = os.environ.get("SOURCE_DATE_EPOCH") + stamp = ( + datetime.fromtimestamp(int(epoch), timezone.utc) + if epoch and epoch.isdigit() + else datetime.now(timezone.utc) + ) + + return { + "spdxVersion": SPDX_VERSION, + "dataLicense": "CC0-1.0", + "SPDXID": "SPDXRef-DOCUMENT", + "name": wheel.path.name.removesuffix(".whl"), + "documentNamespace": f"{project['document_namespace']}/{wheel.path.name}/{code}", + "creationInfo": { + "created": stamp.strftime("%Y-%m-%dT%H:%M:%SZ"), + "creators": [ + f"Tool: {TOOL_NAME}-{TOOL_VERSION}", + f"Organization: {project['supplier'].split(': ', 1)[-1]}", + ], + "licenseListVersion": ".".join( + str(evidence_doc.get("license_list_version", "3.29")).split(".")[:2] + ), + "comment": ( + "Contents inventory for one built wheel, discovered from the build " + "that produced it. Build-time-only inputs, toolchains and " + "source-distribution contents are out of scope here." + ), + }, + "packages": packages, + "files": files, + "relationships": relationships, + "hasExtractedLicensingInfos": _extracted_licenses( + list(inventory.components_present.values()) + ), + } diff --git a/scripts/sbom/sbom/elf.py b/scripts/sbom/sbom/elf.py new file mode 100644 index 0000000000..0a0f7b69f6 --- /dev/null +++ b/scripts/sbom/sbom/elf.py @@ -0,0 +1,68 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""SONAME and NEEDED for the shared libraries a wheel carries. + +Parsing is pyelftools' job, not ours -- it is the same library auditwheel uses +to repair these wheels, so the tool that wrote the ELF and the tool that reads +it agree by construction. + +Only the build path needs any of this. Consumer-side verification re-hashes +files and never parses a binary, which keeps attacker-controlled input out of a +parser on that path entirely. +""" + +from __future__ import annotations + +import io +from dataclasses import dataclass + +from elftools.elf.elffile import ELFFile + +ELF_MAGIC = b"\x7fELF" + +# pyelftools' arch names, mapped to what a wheel's platform tag calls them. +_ARCH = {"x64": "x86_64", "x86": "i386", "AArch64": "aarch64", "ARM": "arm"} + + +@dataclass(frozen=True) +class DynamicInfo: + soname: str | None + needed: tuple[str, ...] + machine: str + + +class NotAnElf(Exception): + """The blob does not start with the ELF magic.""" + + +def is_elf(data: bytes) -> bool: + return data[:4] == ELF_MAGIC + + +def read_dynamic(data: bytes) -> DynamicInfo: + """Parse SONAME and NEEDED out of an in-memory ELF image.""" + if not is_elf(data): + raise NotAnElf("missing ELF magic") + + elffile = ELFFile(io.BytesIO(data)) + machine = elffile.get_machine_arch() + + soname: str | None = None + needed: list[str] = [] + + # The dynamic segment, not the section: a stripped library keeps its program + # headers and loses its section headers, and the loader reads the segment. + for segment in elffile.iter_segments(type="PT_DYNAMIC"): + for tag in segment.iter_tags(): + kind = tag.entry.d_tag + if kind == "DT_SONAME": + soname = tag.soname + elif kind == "DT_NEEDED": + needed.append(tag.needed) + + return DynamicInfo( + soname=soname, + needed=tuple(needed), + machine=_ARCH.get(machine, machine), + ) diff --git a/scripts/sbom/sbom/evidence.py b/scripts/sbom/sbom/evidence.py new file mode 100644 index 0000000000..0a85414f0d --- /dev/null +++ b/scripts/sbom/sbom/evidence.py @@ -0,0 +1,297 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Assemble every discovery index for one configured build tree.""" + +from __future__ import annotations + +import platform +import re +from dataclasses import dataclass +from datetime import datetime, timezone +from pathlib import Path + +from packageurl import PackageURL + +from . import TOOL_NAME, TOOL_VERSION, licensing +from .discovery import ( + ARCHIVE_SUFFIXES, + ArchiveIndex, + Ownership, + git, + BuildGraph, + Component, + FileIndex, + discover_archives, + discover_vendored, + project_authors, + discover_source_trees, +) + +# Cache entries that pin what a wheel contains. The rest of the cache is paths on +# the builder. +_CACHE_KEYS = re.compile( + r"^(CMAKE_BUILD_TYPE|CMAKE_CXX_COMPILER|CMAKE_C_COMPILER|CMAKE_SYSTEM_PROCESSOR" + r"|CMAKE_CXX_COMPILER_VERSION|ISAAC_TELEOP_PYTHON_VERSION|BUILD_[A-Z0-9_]+" + r"|ENABLE_[A-Z0-9_]+|VCPKG_[A-Z0-9_]+)$" +) + + +class EvidenceError(Exception): + """The build tree does not carry the evidence a published wheel needs.""" + + +@dataclass +class Discovery: + """Everything one build tree can tell us, indexed for attribution.""" + + repo_root: Path + build_dir: Path + config: str + cache: dict[str, str] + components: dict[str, Component] + graph: BuildGraph + archives: ArchiveIndex + source_files: FileIndex + repo_files: FileIndex + build_files: FileIndex + ownership: Ownership + + @property + def staged_root(self) -> Path | None: + staged = self.build_dir / "python_package" / self.config + return staged if staged.is_dir() else None + + +def _cmake_cache(build_dir: Path) -> dict[str, str]: + cache = build_dir / "CMakeCache.txt" + if not cache.is_file(): + raise EvidenceError( + f"{cache} not found; point --build-dir at a configured build tree" + ) + entries: dict[str, str] = {} + for line in cache.read_text(encoding="utf-8", errors="replace").splitlines(): + if line.startswith(("#", "//")) or "=" not in line or ":" not in line: + continue + name = line.split(":", 1)[0] + if _CACHE_KEYS.match(name): + entries[name] = line.split("=", 1)[1] + return entries + + +def _archive_components(archives: ArchiveIndex) -> dict[str, Component]: + """A tarball in the repo is a component: its members ship as they are.""" + components: dict[str, Component] = {} + for display, record in archives.archives.items(): + name = Path(display).name + for suffix in ARCHIVE_SUFFIXES: + name = name.removesuffix(suffix) + + evidence = [ + licensing._evidence( # noqa: SLF001 - same package + display, + "archive-member", + f"{display}!{member.path}", + data.decode("utf-8", "replace"), + licensing._classify(Path(member.path).name) or "grant", # noqa: SLF001 + ) + for member, data in record.get("license_members", []) + ] + concluded, declared = licensing.expression(evidence, name) + + components[display] = Component( + key=display, + name=name, + kind="archive", + supplier="NOASSERTION", + homepage="NOASSERTION", + purl=PackageURL(type="generic", name=name).to_string(), + version=_archive_version(archives, display) or "NOASSERTION", + download_location="NOASSERTION", + source_info=( + f"Unpacked from {display} (sha256:{record['sha256']}, " + f"{record['member_count']} members) held in this repository." + ), + license_concluded=concluded, + license_declared=declared, + evidence=tuple(evidence), + ) + return components + + +def _archive_version(archives: ArchiveIndex, display: str) -> str | None: + """SDK tarballs state their version in a root VERSION member, or in the name.""" + stated = archives.archives.get(display, {}).get("version_text") + if stated: + return stated + match = re.search( + r"(\d+\.\d+[\w.+-]*?)(?=-|\.tar|\.tgz|\.zip|$)", Path(display).name + ) + return match.group(1) if match else None + + +def discover(repo_root: Path, build_dir: Path) -> Discovery: + """Index a configured build tree. Nothing here is declared anywhere.""" + cache = _cmake_cache(build_dir) + deps_dir = build_dir / "_deps" + if not deps_dir.is_dir(): + raise EvidenceError( + f"{deps_dir} not found; configure the build before collecting evidence" + ) + + config = cache.get("CMAKE_BUILD_TYPE") or "Release" + graph = BuildGraph(build_dir, config) + components = discover_source_trees(deps_dir) + archives = discover_archives( + repo_root, skip=(build_dir, repo_root / "build-wheel", repo_root / "dist") + ) + components.update(_archive_components(archives)) + + source_files = FileIndex() + for root in sorted(deps_dir.glob("*-src")): + if root.is_dir(): + source_files.add_tree(root, root.name) + + repo_files = FileIndex() + # Skip the build tree by path, not by name: --build-dir is free-form, and a + # differently-named one inside the repo would otherwise be walked twice. + repo_files.add_tree( + repo_root, + ".", + skip=(build_dir, repo_root / "build-wheel", repo_root / "dist"), + ) + + build_files = FileIndex() + build_files.add_tree( + build_dir, "build", skip=(deps_dir, build_dir / "python_package") + ) + + # Third-party code checked in here rather than fetched: the candidates are + # what the build actually compiles or includes, so nothing outside the build + # is inspected. + ownership = Ownership() + for root in sorted(deps_dir.glob("*-src")): + if root.is_dir(): + ownership.register_root(root, root.name.removesuffix("-src")) + + candidates: set[Path] = set() + for node in graph._nodes.values(): # noqa: SLF001 - same package + candidates.update( + source + for source in node.sources + if str(source).startswith(str(repo_root)) + and "/_deps/" not in source.as_posix() + ) + for include in node.includes: + text = include.as_posix() + if not text.startswith(str(repo_root)) or "/_deps/" in text: + continue + if include.is_dir() and not text.startswith(str(build_dir)): + candidates.update(item for item in include.rglob("*") if item.is_file()) + + vendored, owned_paths = discover_vendored( + repo_root, candidates, project_authors(repo_root) + ) + components.update(vendored) + for key, paths in owned_paths.items(): + for path in paths: + ownership.register_file(path, key) + + return Discovery( + repo_root=repo_root, + build_dir=build_dir, + config=config, + cache=cache, + components=components, + graph=graph, + archives=archives, + source_files=source_files, + repo_files=repo_files, + build_files=build_files, + ownership=ownership, + ) + + +def document(discovery: Discovery, wheel_name: str, inventory) -> dict: + """The build-evidence sidecar: resolved facts, not the raw indexes.""" + repo_root = discovery.repo_root + arch = { + "x86_64": "amd64", + "AMD64": "amd64", + "aarch64": "arm64", + "arm64": "arm64", + }.get( + discovery.cache.get("CMAKE_SYSTEM_PROCESSOR", platform.machine()), + platform.machine(), + ) + + artifacts = [] + for artifact in discovery.graph.artifacts.values(): + contributions = discovery.graph.contributions(artifact, discovery.ownership) + if not contributions: + continue + try: + output = artifact.output.relative_to(discovery.build_dir).as_posix() + except ValueError: + output = artifact.output.as_posix() + artifacts.append( + { + "target": artifact.target, + "output": output, + "first_party": discovery.graph.first_party( + artifact, repo_root, discovery.ownership + ), + "contributions": { + key: sorted(how) for key, how in sorted(contributions.items()) + }, + } + ) + + return { + "schema": "isaaccapture-build-evidence/2", + "tool": {"name": TOOL_NAME, "version": TOOL_VERSION}, + "collected_at": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"), + "wheel": wheel_name, + "project": { + "commit": git(repo_root, "rev-parse", "HEAD"), + "describe": git(repo_root, "describe", "--tags", "--always", "--dirty"), + "version_file": (repo_root / "VERSION").read_text(encoding="utf-8").strip() + if (repo_root / "VERSION").is_file() + else None, + }, + "build": { + "build_dir": str(discovery.build_dir), + "config": discovery.config, + "arch": arch, + "platform": platform.platform(), + "cmake_cache": discovery.cache, + }, + "license_data": licensing.corpus_provenance(), + "license_list_version": licensing.corpus()[0], + "components": { + key: { + "name": component.name, + "kind": component.kind, + "supplier": component.supplier, + "homepage": component.homepage, + "purl": component.purl, + "version": component.version, + "download_location": component.download_location, + "source_info": component.source_info, + "license_concluded": component.license_concluded, + "license_declared": component.license_declared, + "evidence": [item.as_json() for item in component.evidence], + "in_this_wheel": key in inventory.components_present, + } + for key, component in sorted(discovery.components.items()) + }, + "archives": { + display: { + key: value for key, value in record.items() if key != "license_members" + } + for display, record in sorted(discovery.archives.archives.items()) + }, + "link_artifacts": sorted(artifacts, key=lambda item: item["output"]), + "attributions": [item.as_json() for item in inventory.attributions.values()], + "excluded_components": inventory.absent_components, + } diff --git a/scripts/sbom/sbom/inventory.py b/scripts/sbom/sbom/inventory.py new file mode 100644 index 0000000000..b324a1b56f --- /dev/null +++ b/scripts/sbom/sbom/inventory.py @@ -0,0 +1,359 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Account for every file in a wheel by where it came from. + +Each member is traced to a link step, a fetched source tree, an archive in the +repo, the working tree, or the build's own generated output. A member that none +of those explain is an error: it means something reached the wheel by a route +this collector cannot see, and it stops publication. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from pathlib import Path +from typing import TYPE_CHECKING + +from . import licensing +from .discovery import ( + Component, + resolve_system_library, + sha256_file, + system_component, +) +from . import wheelfile +from .wheelfile import Entry, WheelInfo + +if TYPE_CHECKING: # pragma: no cover + from .evidence import Discovery + +FIRST_PARTY = "__first_party__" + +# Text-ish members worth reading a REUSE tag out of. +_TAGGABLE = { + ".py", + ".pyi", + ".txt", + ".cfg", + ".toml", + ".yaml", + ".yml", + ".xml", + ".json", + ".md", +} + + +@dataclass +class Attribution: + """One wheel member and the evidence that explains it.""" + + path: str + origin: str + detail: str + components: dict[str, set[str]] = field(default_factory=dict) + primary: str | None = None + spdx_tag: str | None = None + + def as_json(self) -> dict: + return { + "path": self.path, + "origin": self.origin, + "detail": self.detail, + "primary": self.primary, + "components": { + key: sorted(how) for key, how in sorted(self.components.items()) + }, + "spdx_tag": self.spdx_tag, + } + + +@dataclass +class Inventory: + attributions: dict[str, Attribution] + components_present: dict[str, Component] + component_roles: dict[str, set[str]] + external_runtime: dict[str, list[str]] + requires_dist: list[str] + unattributed: list[str] = field(default_factory=list) + absent_components: dict[str, str] = field(default_factory=dict) + + def files_of(self, key: str) -> list[str]: + return sorted( + path for path, item in self.attributions.items() if item.primary == key + ) + + +class Resolver: + """Probes a candidate's bytes and path against every discovered index.""" + + def __init__(self, discovery: Discovery, system_resolver=None) -> None: + self._system_resolver = system_resolver or resolve_system_library + self.system_libraries: dict[str, dict] = {} + self.repo_root = discovery.repo_root + self.build_dir = discovery.build_dir + self.graph = discovery.graph + self.components = discovery.components + self.archives = discovery.archives + self.repo_files = discovery.repo_files + self.build_files = discovery.build_files + self.ownership = discovery.ownership + self.source_files = discovery.source_files + self._artifacts_by_hash: dict[str, list] = {} + for artifact in self.graph.artifacts.values(): + if artifact.output.is_file(): + # build_files already hashed everything under the build tree. + digest = self.build_files.digest_of(artifact.output) or sha256_file( + artifact.output + ) + self._artifacts_by_hash.setdefault(digest, []).append(artifact) + + def _from_artifact(self, artifact, path: str, detail: str) -> Attribution: + contributions = self.graph.contributions(artifact, self.ownership) + first_party = self.graph.first_party(artifact, self.repo_root, self.ownership) + primary = None + if not first_party: + # The component that supplied the compiled sources owns the binary; + # the rest are linked into it. + sources = [ + key for key, how in contributions.items() if "compiled-source" in how + ] + primary = min( + sources, key=lambda key: (key != artifact.target, key), default=None + ) + return Attribution( + path=path, + origin="built", + detail=f"{detail}; linked by target {artifact.target}", + components={key: set(how) for key, how in contributions.items()}, + primary=primary, + ) + + def resolve( + self, + wheel_path: str, + digest: str, + probe: Path | None, + soname: str | None = None, + ) -> Attribution | None: + """Explain one member: by content, then by path, then by the machine.""" + candidate_digest = digest + source = "wheel member" + if probe is not None and probe.is_file(): + candidate_digest = sha256_file(probe) + source = f"staged at {probe.relative_to(self.build_dir).as_posix()}" + + artifacts = self._artifacts_by_hash.get(candidate_digest) + if artifacts: + return self._from_artifact( + artifacts[0], wheel_path, f"{source}; exact build output" + ) + + component_file = self.source_files.by_hash.get(candidate_digest) + if component_file: + key = component_file.split("/", 1)[0].removesuffix("-src") + return Attribution( + path=wheel_path, + origin="copied", + detail=f"{source}; byte-identical to {component_file}", + components={key: {"copied-file"}}, + primary=key, + ) + + member = self.archives.by_hash.get(candidate_digest) + if member: + return Attribution( + path=wheel_path, + origin="archive-copy", + detail=f"{source}; byte-identical to {member.path} in {member.container}", + components={member.container: {"extracted-file"}}, + primary=member.container, + ) + + repo_file = self.repo_files.by_hash.get(candidate_digest) + if repo_file: + return Attribution( + path=wheel_path, origin="repo-source", detail=f"{source}; {repo_file}" + ) + + build_file = self.build_files.by_hash.get(candidate_digest) + if build_file: + return Attribution( + path=wheel_path, origin="generated", detail=f"{source}; {build_file}" + ) + + by_path = self._resolve_by_path(wheel_path, source) + if by_path is not None: + return by_path + + # Nothing this build produced or fetched explains these bytes. A shared + # library reaching that point came from the machine -- whatever the + # repair tool chose to call its directory. + if soname: + return self._from_build_host(wheel_path, soname) + return None + + def _from_build_host(self, wheel_path: str, soname: str) -> Attribution | None: + """Claim host origin only if the host actually has this library. + + Resolving the SONAME is the evidence for the claim. Without it there is + nothing behind "the machine supplied this", and the member is better + reported as unexplained than attributed to a library no one found. + """ + origin = self._system_resolver(soname) + if not origin.get("resolved_path"): + return None + self.system_libraries[soname] = origin + component = system_component(soname, origin) + self.components[component.key] = component + + detail = "no index explains these bytes; supplied by the build machine" + if origin.get("package"): + detail += f" as {origin['package']}" + if origin.get("resolved_path"): + detail += f" ({origin['resolved_path']})" + return Attribution( + path=wheel_path, + origin="build-host-library", + detail=detail, + components={component.key: {"vendored-library"}}, + primary=component.key, + ) + + def _resolve_by_path(self, wheel_path: str, source: str) -> Attribution | None: + """Post-processing changes bytes; the path still says where they came from. + + patchelf, auditwheel's RPATH rewrite and the MJCF mesh stripping all leave + a file that no longer hashes to its origin, so a unique path match is the + remaining evidence -- recorded as derived, never as an exact copy. + """ + name = Path(wheel_path).name + + members = self.archives.by_name.get(name, []) + containers = {member.container for member in members} + if len(containers) == 1: + member = members[0] + return Attribution( + path=wheel_path, + origin="archive-derived", + detail=( + f"{source}; modified copy of {member.path} in {member.container} " + f"(origin sha256:{member.sha256})" + ), + components={member.container: {"extracted-file"}}, + primary=member.container, + ) + + artifacts = self.graph.by_name(name) + if len(artifacts) == 1: + return self._from_artifact( + artifacts[0], wheel_path, f"{source}; modified build output" + ) + + component_file = self.source_files.path_suffix_match(wheel_path) + if component_file: + key = component_file.split("/", 1)[0].removesuffix("-src") + return Attribution( + path=wheel_path, + origin="derived", + detail=f"{source}; derived from {component_file}", + components={key: {"derived-file"}}, + primary=key, + ) + + repo_file = self.repo_files.path_suffix_match(wheel_path) + if repo_file: + return Attribution( + path=wheel_path, + origin="repo-source", + detail=f"{source}; derived from {repo_file}", + ) + + return None + + +def build( + wheel: WheelInfo, + metadata_bytes: bytes, + resolver: Resolver, + staged_root: Path | None, +) -> Inventory: + """Attribute every member, then fold the result into component presence.""" + attributions: dict[str, Attribution] = {} + unattributed: list[str] = [] + provided_sonames = {entry.soname for entry in wheel.entries if entry.soname} + + for entry in wheel.entries: + attribution = _attribute(entry, wheel, resolver, staged_root) + if attribution is None: + unattributed.append(entry.name) + continue + attribution.spdx_tag = _spdx_tag(entry) + attributions[entry.name] = attribution + + roles: dict[str, set[str]] = {} + for attribution in attributions.values(): + for key, how in attribution.components.items(): + roles.setdefault(key, set()).update(how) + + present = { + key: resolver.components[key] + for key in sorted(roles) + if key in resolver.components + } + absent = { + key: "fetched for this build; contributes to no file in this wheel" + for key in sorted(resolver.components) + if key not in roles + } + + external: dict[str, list[str]] = {} + for entry in wheel.entries: + for soname in entry.needed: + if soname not in provided_sonames: + external.setdefault(soname, []).append(entry.name) + + metadata = wheelfile.parse_metadata(metadata_bytes) + return Inventory( + attributions=attributions, + components_present=present, + component_roles=roles, + external_runtime={ + name: sorted(set(files)) for name, files in sorted(external.items()) + }, + requires_dist=sorted(metadata.get("requires_dist") or []), + unattributed=sorted(unattributed), + absent_components=absent, + ) + + +def _attribute( + entry: Entry, + wheel: WheelInfo, + resolver: Resolver, + staged_root: Path | None, +) -> Attribution | None: + if entry.name.startswith(f"{wheel.dist_info}/"): + return Attribution( + path=entry.name, + origin="metadata", + detail="distribution metadata written by the build", + ) + + probe = staged_root / entry.name if staged_root is not None else None + return resolver.resolve(entry.name, entry.sha256, probe, entry.soname) + + +def _spdx_tag(entry: Entry) -> str | None: + """REUSE tags are upstream stating the license of that exact file.""" + if Path(entry.name).suffix.lower() not in _TAGGABLE: + return None + tag = licensing.read_spdx_tag(entry.head.decode("utf-8", "replace")) + if tag is None: + return None + # Tags are written by hand: lowercase operators and stray text reach the + # document as licenseConcluded, where an invalid expression fails the + # published SBOM's own conformance check. + normalized = licensing.normalized_expression(tag) + return None if normalized == "NOASSERTION" else normalized diff --git a/scripts/sbom/sbom/licensing.py b/scripts/sbom/sbom/licensing.py new file mode 100644 index 0000000000..36d38f3538 --- /dev/null +++ b/scripts/sbom/sbom/licensing.py @@ -0,0 +1,505 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Find the license texts a component ships, and name what they are. + +Nothing here declares a license. Texts are discovered in the component's own +files and matched against the SPDX reference corpus; a text that matches nothing +is still packaged and reported as unidentified, never guessed at. +""" + +from __future__ import annotations + +import hashlib +import json +import re +from functools import lru_cache +from dataclasses import dataclass + +from license_expression import ( + ExpressionError, + combine_expressions, + get_spdx_licensing, +) +from pathlib import Path + + +# Root-level names that carry a grant, and the ones that carry notices instead. +# Both are packaged; only the first kind is matched against the corpus. +# The word has to stand alone: BSD-LICENSE and LICENSE.md are grants, while +# licenses.md and accessingLicenses.md are indexes that merely mention them. +_GRANT_NAMES = re.compile( + r"(^|[-_.])(LICEN[CS]E|COPYING|UNLICENSE|EULA)([-_.]|$)", re.IGNORECASE +) +_NOTICE_NAMES = re.compile(r"^(NOTICE|COPYRIGHT|AUTHORS|PATENTS)", re.IGNORECASE) +_README_NAMES = re.compile(r"^README", re.IGNORECASE) +# Skip machine-readable siblings; the text beside them is the evidence. +_SKIP_SUFFIXES = {".py", ".cmake", ".in", ".json", ".yaml", ".yml", ".spdx"} + +_WORD = re.compile(r"[^a-z0-9]+") +_NGRAM = 6 +# A canonical text has to be almost entirely present before it is named. +_CONTAINMENT_THRESHOLD = 0.90 +# ...and has to account for a real share of the file it was found in. Large +# agreements (the CUDA EULA, for one) reproduce whole OSS licenses in an +# appendix; those reach full containment at about 3% coverage, while a genuine +# match -- even a Debian copyright file wrapped in packaging boilerplate -- sits +# at 30% or above. +_COVERAGE_THRESHOLD = 0.20 +# Two licenses are one family when either text is nearly inside the other. +_FAMILY_THRESHOLD = 0.90 +# REUSE-IgnoreStart +_SPDX_TAG = re.compile(r"SPDX-License-Identifier:\s*(?P[^\n\r*/#]+)") +# REUSE-IgnoreEnd +# Heading, then everything up to the next heading or a blank-line run. +# The heading has to BE about licensing, not merely start with the word -- +# "# License List Data" is a project title, not a grant. +_README_LICENSE_SECTION = re.compile( + r"^#{1,6}\s*(licen[cs]e|licen[cs]es|licensing|licen[cs]e information|copyright|legal)" + r"\s*$(?P.*?)(?=^#{1,6}\s|\Z)", + re.IGNORECASE | re.MULTILINE | re.DOTALL, +) + + +@dataclass(frozen=True) +class LicenseEvidence: + """A license text that was read out of the component, not assumed.""" + + component: str + origin: str # "component-file" | "archive-member" | "readme-section" | "build-host" + path: str + sha256: str + size: int + text: str + # "grant" is the component's own license, "notice" an attribution file, and + # "pool" a REUSE LICENSES/ entry that applies to individual files only. + kind: str + identified: str | None = None + matches: tuple[tuple[str, float, float], ...] = () + spdx_tag: str | None = None + + def as_json(self) -> dict: + return { + "component": self.component, + "origin": self.origin, + "path": self.path, + "sha256": self.sha256, + "size": self.size, + "kind": self.kind, + "identified": self.identified, + "matches": [ + { + "license": license_id, + "containment": round(containment, 4), + "coverage": round(coverage, 4), + } + for license_id, containment, coverage in self.matches + ], + "spdx_tag": self.spdx_tag, + } + + +_DIGITS = re.compile(r"\d+") + + +def _normalize(text: str) -> list[str]: + """Reduce a license to comparable boilerplate. + + Years and clause numbers are the part that differs between two copies of one + license, so digits come out. Copyright lines stay: BSD clause 3 and MIT's + notice-retention clause both contain the word, and upstream wrapping differs + from SPDX's, so dropping those lines loses real text asymmetrically. + """ + return [ + word for word in _WORD.sub(" ", _DIGITS.sub(" ", text.lower())).split() if word + ] + + +def _ngrams(words: list[str]) -> set[str]: + if len(words) < _NGRAM: + return {" ".join(words)} if words else set() + return { + " ".join(words[index : index + _NGRAM]) + for index in range(len(words) - _NGRAM + 1) + } + + +@dataclass(frozen=True) +class Reference: + """One SPDX licence, normalized for comparison.""" + + name: str + grams: frozenset[str] + + @property + def size(self) -> int: + return len(self.grams) + + +class CorpusError(Exception): + """The SPDX reference texts this build fetched are missing or unreadable.""" + + +_CORPUS: tuple[str, dict[str, Reference]] | None = None +_CORPUS_META: dict[str, str] = {} +# Parsing and n-gramming the whole SPDX list costs about half a second. Keyed by +# directory and only ever read within one process, so a re-load is free. +_PARSED: dict[str, tuple[str, dict[str, Reference]]] = {} + + +def corpus_provenance() -> dict[str, str]: + """Which reference texts an identification was made against.""" + return dict(_CORPUS_META) + + +def load_corpus(json_dir: Path) -> str: + """Load SPDX reference texts from the checkout deps/third_party fetched. + + `json_dir` is the `json/` directory of spdx/license-list-data. Deprecated + identifiers are skipped: naming a component with one would be reporting a + license SPDX has withdrawn. + """ + global _CORPUS + details = json_dir / "details" + index_file = json_dir / "licenses.json" + if not details.is_dir() or not index_file.is_file(): + raise CorpusError( + f"SPDX license list data not found at {json_dir}. It is fetched by the " + "build (deps/third_party); configure and build, or pass --license-data." + ) + + cached = _PARSED.get(str(details.resolve())) + if cached is not None: + version, entries = cached + else: + entries = {} + for path in sorted(details.glob("*.json")): + try: + payload = json.loads(path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as error: + raise CorpusError(f"{path} could not be read: {error}") from error + if payload.get("isDeprecatedLicenseId"): + continue + grams = _ngrams(_normalize(payload.get("licenseText", ""))) + if grams: + entries[payload["licenseId"]] = Reference( + name=payload.get("name", payload["licenseId"]), + grams=frozenset(grams), + ) + + if not entries: + raise CorpusError(f"no usable license texts under {details}") + + version = json.loads(index_file.read_text(encoding="utf-8")).get( + "licenseListVersion", "unknown" + ) + _PARSED[str(details.resolve())] = (version, entries) + + _CORPUS = (version, entries) + _CORPUS_META.clear() + _CORPUS_META.update( + { + "license_list_version": version, + "source": str(json_dir), + "commit": _checkout_commit(json_dir.parent), + "licenses": str(len(entries)), + } + ) + return version + + +def _checkout_commit(source_dir: Path) -> str: + """The commit deps/third_party pinned, read back from the checkout itself.""" + from .discovery import git + + return git(source_dir, "rev-parse", "HEAD") or "NOASSERTION" + + +def corpus() -> tuple[str, dict[str, Reference]]: + """SPDX reference texts as (list version, {id: (name, n-grams)}).""" + if _CORPUS is None: + raise CorpusError( + "SPDX license list data has not been loaded; it is fetched by the build " + "into _deps/license-list-data-src (see deps/third_party)" + ) + return _CORPUS + + +def identify(text: str) -> list[tuple[str, float, float]]: + """Name the licenses a text contains, as (id, containment, coverage). + + Containment says how much of a canonical license is present; coverage says + how much of this file that accounts for. Both must clear their threshold, so + a license quoted in an appendix is not mistaken for the file's own terms. A + file really can carry two licenses -- dual-licensed sources do -- so this + returns every match that survives the family resolution below. + + Every reference is compared. The thresholds do bound how long a reference + can be relative to the file, so matches could be skipped on length -- but + that ties correctness to the accept conditions never growing, and it fails + by quietly not identifying a license. Keep the comparison exhaustive. + """ + candidate = _ngrams(_normalize(text)) + if not candidate: + return [] + + entries = corpus()[1] + scored: dict[str, tuple[float, float, float]] = {} + for license_id, reference in entries.items(): + overlap = len(reference.grams & candidate) + if not overlap: + continue + containment = overlap / reference.size + coverage = overlap / len(candidate) + if containment >= _CONTAINMENT_THRESHOLD and coverage >= _COVERAGE_THRESHOLD: + scored[license_id] = ( + containment, + coverage, + overlap / len(reference.grams | candidate), + ) + + dominated = _dominated(scored, entries) + return sorted( + (license_id, containment, coverage) + for license_id, (containment, coverage, _) in scored.items() + if license_id not in dominated + ) + + +def _dominated( + scored: dict[str, tuple[float, float, float]], + entries: dict[str, Reference], +) -> set[str]: + """Within a family of near-identical licenses, keep only the best fit. + + Whole families differ by one clause: BSD-2 inside BSD-3, Apache-2.0 inside + ECL-2.0, CC-BY-4.0 inside CC-BY-NC-4.0. Every one of those clears the + containment threshold against a file that is really the other, so the + discriminator has to be which reference explains the whole file *and nothing + more* -- the highest Jaccard similarity. Preferring the longer text instead + would report Apache-2.0 code as ECL-2.0, and CC-BY-4.0 as its + non-commercial variant. + """ + dropped: set[str] = set() + for left in scored: + for right in scored: + if left == right or right in dropped: + continue + shared = len(entries[left].grams & entries[right].grams) + related = shared / min(entries[left].size, entries[right].size) + if related >= _FAMILY_THRESHOLD and scored[right][2] > scored[left][2]: + dropped.add(left) + break + return dropped + + +def _read(path: Path) -> str | None: + try: + data = path.read_bytes() + except OSError: + return None + if not data.strip() or b"\0" in data[:4096]: + return None + return data.decode("utf-8", "replace") + + +def _evidence( + component: str, origin: str, display: str, text: str, kind: str +) -> LicenseEvidence: + matchable = kind == "grant" + tag = _SPDX_TAG.search(text[:4096]) if matchable else None + matches = tuple(identify(text)) if matchable else () + identified = " AND ".join(license_id for license_id, _, _ in matches) or None + return LicenseEvidence( + component=component, + origin=origin, + path=display, + sha256=hashlib.sha256(text.encode("utf-8")).hexdigest(), + size=len(text.encode("utf-8")), + text=text, + kind=kind, + identified=identified, + matches=matches, + spdx_tag=tag.group("expression").strip() if tag else None, + ) + + +def _classify(name: str) -> str | None: + if Path(name).suffix.lower() in _SKIP_SUFFIXES: + return None + if _GRANT_NAMES.search(name): + return "grant" + if _NOTICE_NAMES.search(name): + return "notice" + return None + + +def discover_in_tree( + component: str, root: Path, display_prefix: str +) -> list[LicenseEvidence]: + """Every license and notice file a source tree carries, at its root or in LICENSES/.""" + found: list[LicenseEvidence] = [] + candidates: list[tuple[Path, str]] = [] + + for entry in sorted(root.iterdir()) if root.is_dir() else []: + if entry.is_file() and (kind := _classify(entry.name)): + candidates.append((entry, kind)) + elif entry.is_dir() and entry.name.upper() in {"LICENSES", "LICENSE"}: + candidates.extend( + (item, "pool") for item in sorted(entry.rglob("*")) if item.is_file() + ) + + for path, kind in candidates: + text = _read(path) + if text is None: + continue + display = f"{display_prefix}/{path.relative_to(root).as_posix()}" + found.append(_evidence(component, "component-file", display, text, kind)) + + if not any(item.kind == "grant" for item in found): + found.extend(_readme_fallback(component, root, display_prefix)) + return found + + +def _readme_fallback( + component: str, root: Path, display_prefix: str +) -> list[LicenseEvidence]: + """Some upstreams state the grant in a README section and ship no license file.""" + for entry in sorted(root.iterdir()) if root.is_dir() else []: + if not entry.is_file() or not _README_NAMES.match(entry.name): + continue + text = _read(entry) + if text is None: + continue + section = _README_LICENSE_SECTION.search(text) + if section is None or not section.group("body").strip(): + continue + display = f"{display_prefix}/{entry.name}" + return [_evidence(component, "readme-section", display, text, "grant")] + return [] + + +_ID_SAFE = re.compile(r"[^A-Za-z0-9.-]+") + + +def spdx_safe(value: str) -> str: + """Reduce a string to what an SPDX identifier may contain.""" + return _ID_SAFE.sub("-", value).strip("-") + + +def license_ref(component_key: str) -> str: + """Identifier for a grant that is real but matches no SPDX reference text.""" + return f"LicenseRef-{spdx_safe(component_key)}" + + +def expression( + evidence: list[LicenseEvidence], component_key: str | None = None +) -> tuple[str, str]: + """Concluded and declared expressions for one component. + + Concluded names what was matched in a text that ships with the artifact. A + grant that matches nothing is not dropped and not guessed at: it becomes a + LicenseRef whose text travels in the document, which says "these are the + terms we shipped" without claiming to know which license they are. + """ + grants = [item for item in evidence if item.kind == "grant"] + identified = sorted( + {license_id for item in grants for license_id, _, _ in item.matches} + ) + if not identified and grants and component_key: + identified = [license_ref(component_key)] + concluded = _combine(identified) if identified else "NOASSERTION" + + tags = sorted({item.spdx_tag for item in grants if item.spdx_tag}) + declared = _combine(tags) if tags else concluded + return concluded, declared + + +def _combine(expressions: list[str]) -> str: + """Join with AND through the SPDX grammar rather than by string join.""" + try: + combined = combine_expressions( + sorted(expressions), relation="AND", licensing=_spdx_licensing() + ) + except (ExpressionError, ValueError, TypeError): + return "NOASSERTION" + # `is None`, not truthiness: a parsed expression raises on __bool__. + return "NOASSERTION" if combined is None else str(combined) + + +def normalized_expression(expression: str) -> str: + """Put an expression through the SPDX grammar before it reaches a document. + + Upstream tags are written by hand and arrive with lowercase operators and + odd spacing; emitting those unchanged would put an unparseable expression in + an SBOM. Anything the grammar rejects is reported as NOASSERTION rather than + passed through. + """ + try: + return str(_spdx_licensing().parse(expression, validate=False)) + except (ExpressionError, ValueError, TypeError): + return "NOASSERTION" + + +def read_spdx_tag(text: str) -> str | None: + """The REUSE tag a file states about itself, if any.""" + match = _SPDX_TAG.search(text) + return match.group("expression").strip() if match else None + + +def combine(expressions: list[str]) -> str: + """Public form of the AND-combiner, for callers assembling an expression.""" + return _combine(expressions) + + +def pool_evidence( + repo_root: Path, component_key: str, expressions: set[str] +) -> list[LicenseEvidence]: + """Texts from the repository's REUSE pool for the ids a file declares. + + A vendored file states an identifier and nothing else; LICENSES/ is where + this repository already keeps the matching text, which is what the pool is + for. + """ + found: list[LicenseEvidence] = [] + seen: set[str] = set() + for expression in sorted(expressions): + for token in _spdx_licensing().license_keys( + _spdx_licensing().parse(expression, validate=False) + ): + candidate = repo_root / "LICENSES" / f"{token}.txt" + if token in seen or not candidate.is_file(): + continue + seen.add(token) + found.append( + _evidence( + component_key, + "reuse-pool", + f"LICENSES/{token}.txt", + candidate.read_text(encoding="utf-8", errors="replace"), + "grant", + ) + ) + return found + + +def license_refs(*expressions: str) -> list[str]: + """The LicenseRef- identifiers an expression uses, per the SPDX grammar.""" + found: set[str] = set() + for expression in expressions: + try: + parsed = _spdx_licensing().parse(expression, validate=False) + except (ExpressionError, ValueError, TypeError): + continue + found.update( + key + for key in _spdx_licensing().license_keys(parsed) + if key.startswith("LicenseRef-") + ) + return sorted(found) + + +@lru_cache(maxsize=1) +def _spdx_licensing(): + """Building this parses ScanCode's licence database; do it once.""" + return get_spdx_licensing() diff --git a/scripts/sbom/sbom/report.py b/scripts/sbom/sbom/report.py new file mode 100644 index 0000000000..5adff7dbe6 --- /dev/null +++ b/scripts/sbom/sbom/report.py @@ -0,0 +1,215 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Component-to-license report, and the notices packaged inside the wheel.""" + +from __future__ import annotations + +import re + +from .inventory import Inventory + +_HOW_PROSE = { + "compiled-source": "its sources were compiled into a shipped binary", + "header-include": "its headers were compiled into a shipped binary", + "prebuilt-library": "a pre-built library of its was linked in", + "copied-file": "files were copied in unchanged", + "derived-file": "files were copied in and modified by the build", + "extracted-file": "files were unpacked from its archive", + "vendored-library": "the library was vendored in by auditwheel", +} +_SAFE_NAME = re.compile(r"[^A-Za-z0-9._-]") + + +def packaged_license_path(dist_info: str, component_key: str, evidence) -> str: + """Where one component's license text lands inside the wheel. + + Named after the evidence's whole path within its component, not its + basename: a REUSE pool makes `LICENSE` and `LICENSES/MIT.txt` collide, and + dropping one would leave the notices pointing at a text that is not there. + """ + folder = _SAFE_NAME.sub("_", component_key) + relative = evidence.path.rsplit("!", 1)[-1].split("/", 1)[-1] + return f"{dist_info}/licenses/third-party/{folder}/{_SAFE_NAME.sub('_', relative)}" + + +def _roles(inventory: Inventory, key: str) -> str: + hows = sorted(inventory.component_roles.get(key, set())) + return "; ".join(_HOW_PROSE.get(how, how) for how in hows) or "unrecorded" + + +def notices_markdown(inventory: Inventory, dist_info: str, wheel_name: str) -> str: + """The notice file that travels inside the wheel.""" + lines = [ + "# Third-party notices", + "", + f"Components redistributed in `{wheel_name}`, with the license text each", + "obligation was read from. Texts are packaged under", + f"`{dist_info}/licenses/third-party/`.", + "", + ] + for key, component in inventory.components_present.items(): + lines.append(f"## {component.name}") + lines.append("") + lines.append(f"- License: `{component.license_concluded}`") + if component.license_declared != component.license_concluded: + lines.append(f"- Declared: `{component.license_declared}`") + if component.homepage != "NOASSERTION": + lines.append(f"- Homepage: {component.homepage}") + if component.version != "NOASSERTION": + lines.append(f"- Version: `{component.version}`") + lines.append(f"- Reaches this wheel because {_roles(inventory, key)}") + for item in component.evidence: + lines.append(f"- Text: `{packaged_license_path(dist_info, key, item)}`") + if not component.evidence: + lines.append("- No license text was found in this component's own files.") + lines.append("") + return "\n".join(lines) + + +def report(inventory: Inventory, evidence_doc: dict, wheel_name: str) -> dict: + """Machine-readable component-to-license report, including what is missing.""" + components = [] + no_evidence = [] + unidentified = [] + + for key, component in inventory.components_present.items(): + grants = [item for item in component.evidence if item.kind == "grant"] + components.append( + { + "component": key, + "name": component.name, + "kind": component.kind, + "version": component.version, + "how_it_reaches_the_wheel": sorted( + inventory.component_roles.get(key, set()) + ), + "license_concluded": component.license_concluded, + "license_declared": component.license_declared, + "supplier": component.supplier, + "homepage": component.homepage, + "purl": component.purl, + "source_info": component.source_info, + "wheel_files": inventory.files_of(key), + "evidence": [item.as_json() for item in component.evidence], + } + ) + if not grants: + no_evidence.append( + { + "component": key, + "reason": "no license grant found in the component's own files", + } + ) + elif not any(item.identified for item in grants): + unidentified.append( + { + "component": key, + "reason": ( + "the shipped grant matches no text in the SPDX reference " + "corpus; it is reproduced verbatim in the document" + ), + "evidence": [item.path for item in grants], + } + ) + + return { + "schema": "isaaccapture-license-report/2", + "wheel": wheel_name, + "generated_from_commit": evidence_doc.get("project", {}).get("commit"), + "spdx_license_list_version": evidence_doc.get("license_list_version"), + "license_data": evidence_doc.get("license_data", {}), + "components": components, + "externally_supplied_runtime": [ + {"soname": soname, "required_by": consumers} + for soname, consumers in inventory.external_runtime.items() + ], + "consumer_requirements": inventory.requires_dist, + "exclusions": [ + {"component": key, "reason": reason} + for key, reason in sorted(inventory.absent_components.items()) + ], + "components_without_license_evidence": no_evidence, + "components_with_unidentified_license": unidentified, + } + + +def report_markdown(payload: dict) -> str: + """Human-readable twin of :func:`report`.""" + lines = [ + f"# Component-to-license report — `{payload['wheel']}`", + "", + f"Source commit: `{payload['generated_from_commit'] or 'unknown'}` ", + f"SPDX license list: `{payload['spdx_license_list_version'] or 'unknown'}` " + f"(spdx/license-list-data `{payload.get('license_data', {}).get('commit', '?')[:12]}`)", + "", + "Every row was discovered from the build that produced this wheel. Nothing", + "here is declared in a checked-in list.", + "", + "## Redistributed components", + "", + "| Component | Version | License | How it reaches the wheel | Evidence |", + "| --- | --- | --- | --- | --- |", + ] + for item in payload["components"]: + evidence = ( + "
".join( + f"`{record['path']}`" + + (f" → {record['identified']}" if record["identified"] else "") + for record in item["evidence"] + ) + or "—" + ) + how = "; ".join( + _HOW_PROSE.get(value, value) for value in item["how_it_reaches_the_wheel"] + ) + version = item["version"] + lines.append( + f"| {item['name']} | `{version[:12]}` | `{item['license_concluded']}` | " + f"{how or '—'} | {evidence} |" + ) + + lines += ["", "## Externally supplied at runtime", ""] + if payload["externally_supplied_runtime"]: + lines += [ + "Resolved from the installation environment; not redistributed here.", + "", + ] + lines += [ + f"- `{item['soname']}` — required by {', '.join(item['required_by'])}" + for item in payload["externally_supplied_runtime"] + ] + else: + lines.append("None recorded.") + + lines += ["", "## Consumer requirements (wheel metadata)", ""] + lines += [f"- `{item}`" for item in payload["consumer_requirements"]] or [ + "None declared." + ] + + lines += ["", "## Fetched for this build but not in this wheel", ""] + if payload["exclusions"]: + lines += [ + f"- `{item['component']}` — {item['reason']}" + for item in payload["exclusions"] + ] + else: + lines.append("None.") + + lines += ["", "## License evidence gaps", ""] + gaps = payload["components_without_license_evidence"] + unidentified = payload["components_with_unidentified_license"] + if gaps: + lines += ["No license text found for:", ""] + lines += [f"- `{item['component']}` — {item['reason']}" for item in gaps] + lines.append("") + if unidentified: + lines += ["License text shipped but not identified:", ""] + lines += [ + f"- `{item['component']}` — {item['reason']}" for item in unidentified + ] + lines.append("") + if not gaps and not unidentified: + lines.append("None; every redistributed component ships an identified license.") + + return "\n".join(lines) + "\n" diff --git a/scripts/sbom/sbom/validate.py b/scripts/sbom/sbom/validate.py new file mode 100644 index 0000000000..cdce3b98d7 --- /dev/null +++ b/scripts/sbom/sbom/validate.py @@ -0,0 +1,305 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Checks that decide whether a wheel may be published. + +`verify_wheel` is the consumer-side half: it needs the wheel and nothing else, +so the instructions published with a release are the same code that gates it. +""" + +from __future__ import annotations + +import hashlib +import json +import zipfile +from pathlib import Path + +from . import SPDX_VERSION, licensing +from . import document as document_module +from . import wheelfile + +_REQUIRED_KEYS = ( + "spdxVersion", + "dataLicense", + "SPDXID", + "name", + "documentNamespace", + "creationInfo", + "packages", + "files", + "relationships", +) + + +def _embedded_sbom( + archive: zipfile.ZipFile, dist_info: str, wheel_name: str +) -> tuple[str, dict]: + name = f"{dist_info}/sboms/{wheel_name}.spdx.json" + names = [ + item for item in archive.namelist() if item.startswith(f"{dist_info}/sboms/") + ] + if names != [name]: + raise LookupError( + f"expected exactly {name} under {dist_info}/sboms/, found {names or 'nothing'}" + ) + return name, json.loads(archive.read(name)) + + +def verify_wheel( + wheel_path: Path, scanned: wheelfile.WheelInfo | None = None +) -> list[str]: + """Verify a wheel against the SBOM it carries. No repository required.""" + failures: list[str] = [] + wheel = scanned or wheelfile.scan(wheel_path) + + with zipfile.ZipFile(wheel_path) as archive: + try: + sbom_name, spdx = _embedded_sbom(archive, wheel.dist_info, wheel_path.name) + except (LookupError, json.JSONDecodeError) as error: + return [f"embedded SBOM: {error}"] + + for key in _REQUIRED_KEYS: + if key not in spdx: + failures.append(f"SPDX document is missing required field {key!r}") + if spdx.get("spdxVersion") != SPDX_VERSION: + failures.append( + f"spdxVersion is {spdx.get('spdxVersion')!r}, expected {SPDX_VERSION!r}" + ) + if failures: + return failures + + identifiers = [spdx["SPDXID"]] + identifiers += [item["SPDXID"] for item in spdx["packages"]] + identifiers += [item["SPDXID"] for item in spdx["files"]] + duplicates = sorted({item for item in identifiers if identifiers.count(item) > 1}) + if duplicates: + failures.append(f"duplicate SPDXIDs: {duplicates}") + known = set(identifiers) + for relationship in spdx["relationships"]: + for side in ("spdxElementId", "relatedSpdxElement"): + if relationship[side] not in known: + failures.append( + f"relationship {relationship['relationshipType']} references unknown " + f"element {relationship[side]!r}" + ) + + wheel_package = next( + ( + item + for item in spdx["packages"] + if item["SPDXID"] == "SPDXRef-Package-wheel" + ), + None, + ) + if wheel_package is None: + return failures + ["SPDXRef-Package-wheel is missing"] + if wheel_package.get("packageFileName") != wheel_path.name: + failures.append( + f"packageFileName {wheel_package.get('packageFileName')!r} does not name this wheel" + ) + if ( + f"{wheel_package.get('name')}-{wheel_package.get('versionInfo')}" + != wheel.dist_info.removesuffix(".dist-info") + ): + failures.append( + "wheel package name/version do not match the wheel's .dist-info" + ) + + excluded = { + item.removeprefix("./") + for item in wheel_package.get("packageVerificationCode", {}).get( + "packageVerificationCodeExcludedFiles", [] + ) + } + if sbom_name not in excluded: + failures.append("the embedded SBOM must be listed among the excluded files") + + documented = {item["fileName"].removeprefix("./"): item for item in spdx["files"]} + actual = {item.name: item for item in wheel.entries} + + for name in sorted(set(actual) - set(documented) - excluded): + failures.append(f"wheel file not covered by the SBOM: {name}") + for name in sorted(set(documented) - set(actual)): + failures.append(f"SBOM documents a file the wheel does not contain: {name}") + + sha1_digests = [] + for name, record in sorted(documented.items()): + entry = actual.get(name) + if entry is None: + continue + checksums = { + item["algorithm"]: item["checksumValue"] for item in record["checksums"] + } + if checksums.get("SHA256") != entry.sha256: + failures.append(f"SHA256 mismatch for {name}") + if checksums.get("SHA1") != entry.sha1: + failures.append(f"SHA1 mismatch for {name}") + sha1_digests.append(entry.sha1) + + expected_code = document_module.verification_code(sha1_digests) + actual_code = wheel_package.get("packageVerificationCode", {}).get( + "packageVerificationCodeValue" + ) + if expected_code != actual_code: + failures.append( + f"packageVerificationCode is {actual_code!r} but the documented files hash to {expected_code!r}" + ) + + failures.extend(_check_record(wheel_path, wheel)) + failures.extend(_check_licenses(wheel, spdx)) + return failures + + +def _check_record(wheel_path: Path, wheel: wheelfile.WheelInfo) -> list[str]: + """RECORD is what an installer trusts; a wrong entry here is a broken wheel.""" + failures: list[str] = [] + record = wheelfile.read_record(wheel_path, wheel.dist_info) + record_name = f"{wheel.dist_info}/RECORD" + for entry in wheel.entries: + if entry.name == record_name: + continue + row = record.get(entry.name) + if row is None: + failures.append(f"RECORD does not list {entry.name}") + continue + # scan() already hashed every member; RECORD states the same digest in + # base64, so re-reading the archive to recompute it is wasted work. + if row[0] != wheelfile.record_hash_from_sha256(entry.sha256): + failures.append(f"RECORD digest mismatch for {entry.name}") + if row[1] != str(entry.size): + failures.append(f"RECORD size mismatch for {entry.name}") + for name in sorted(set(record) - {item.name for item in wheel.entries}): + failures.append(f"RECORD lists a file the wheel does not contain: {name}") + return failures + + +def _check_attribution(evidence_doc: dict, wheel: wheelfile.WheelInfo) -> list[str]: + """Every wheel member must name where it came from, and every shipped + component must have had a license text to read.""" + failures: list[str] = [] + attributed = {item["path"] for item in evidence_doc.get("attributions", [])} + + for entry in wheel.entries: + if entry.name not in attributed: + failures.append( + f"build evidence does not explain where {entry.name} came from" + ) + + for key, component in evidence_doc.get("components", {}).items(): + if not component.get("in_this_wheel"): + continue + if not any(item["kind"] == "grant" for item in component.get("evidence", [])): + failures.append( + f"{key} is redistributed but no license text was found for it" + ) + return failures + + +def _check_licenses(wheel: wheelfile.WheelInfo, spdx: dict) -> list[str]: + failures: list[str] = [] + names = {item.name for item in wheel.entries} + notices = f"{wheel.dist_info}/licenses/THIRD-PARTY-NOTICES.md" + if notices not in names: + failures.append(f"{notices} is not packaged") + + declared_refs = set() + for item in spdx.get("files", []): + declared_refs.update( + licensing.license_refs(str(item.get("licenseConcluded", ""))) + ) + for package in spdx["packages"]: + for field in ("licenseConcluded", "licenseDeclared"): + for token in ( + str(package.get(field, "")).replace("(", " ").replace(")", " ").split() + ): + if token.startswith("LicenseRef-"): + declared_refs.add(token) + extracted = { + item["licenseId"] for item in spdx.get("hasExtractedLicensingInfos", []) + } + for missing in sorted(declared_refs - extracted): + failures.append( + f"{missing} is used but has no hasExtractedLicensingInfos entry" + ) + for item in spdx.get("hasExtractedLicensingInfos", []): + if not item.get("extractedText", "").strip(): + failures.append(f"{item['licenseId']} has an empty extractedText") + return failures + + +def check( + wheel: Path, + build_evidence: Path | None = None, + manifest: Path | None = None, + evidence_dir: Path | None = None, +) -> list[str]: + """Check a wheel, and as much of its published evidence as you can supply. + + One operation over widening scope rather than three commands: a consumer has + only the wheel, a release build also has the evidence it just produced, and a + release has the whole set. Each input adds checks; none replaces the others. + """ + scanned = wheelfile.scan(wheel) + failures = verify_wheel(wheel, scanned) + + if build_evidence is not None: + evidence_doc = json.loads(build_evidence.read_text(encoding="utf-8")) + failures.extend(_check_attribution(evidence_doc, scanned)) + + if manifest is not None: + failures.extend(_check_manifest(manifest, wheel, evidence_dir)) + + return failures + + +def _check_manifest( + manifest_path: Path, wheel: Path, evidence_dir: Path | None +) -> list[str]: + """The manifest binds a wheel's digest and names the sidecars beside it.""" + failures: list[str] = [] + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + record = next( + (item for item in manifest["wheels"] if item["filename"] == wheel.name), None + ) + if record is None: + return [f"{manifest_path.name} has no entry for {wheel.name}"] + + digest = hashlib.sha256(wheel.read_bytes()).hexdigest() + if record["sha256"] != digest: + failures.append( + "the wheel was modified after its digest was bound: manifest says " + f"{record['sha256']}, the file hashes to {digest}" + ) + + beside = evidence_dir if evidence_dir is not None else manifest_path.parent + for kind, sidecar in record["sidecars"].items(): + path = beside / sidecar["filename"] + if not path.is_file(): + failures.append(f"sidecar {kind} is missing: {sidecar['filename']}") + elif hashlib.sha256(path.read_bytes()).hexdigest() != sidecar["sha256"]: + failures.append(f"sidecar {kind} does not match its recorded digest") + + published = beside / record["sidecars"]["spdx"]["filename"] + if published.is_file(): + with zipfile.ZipFile(wheel) as archive: + if archive.read(record["sbom_in_wheel"]) != published.read_bytes(): + failures.append( + "the published SBOM copy differs from the one embedded in the wheel" + ) + return failures + + +def check_set(manifest_path: Path, wheel_dir: Path, evidence_dir: Path) -> list[str]: + """Run :func:`check` over every wheel a merged manifest advertises.""" + failures: list[str] = [] + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + for record in manifest["wheels"]: + wheel = wheel_dir / record["filename"] + if not wheel.is_file(): + failures.append(f"{record['filename']} is missing from {wheel_dir}") + continue + failures.extend( + f"{record['filename']}: {item}" + for item in check(wheel, manifest=manifest_path, evidence_dir=evidence_dir) + ) + return failures diff --git a/scripts/sbom/sbom/wheelfile.py b/scripts/sbom/sbom/wheelfile.py new file mode 100644 index 0000000000..50be7bc1c4 --- /dev/null +++ b/scripts/sbom/sbom/wheelfile.py @@ -0,0 +1,280 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Read a wheel's contents and rewrite it with the evidence it must carry. + +RECORD and METADATA are specified formats, so their own tooling parses them: +``installer`` (PyPA) owns RECORD's hashing and escaping rules, and ``packaging`` +owns core metadata. Only the zip rewriting is ours, because no library offers it. +""" + +from __future__ import annotations + +import base64 +import csv +import hashlib +import io +import os +import time +import zipfile +from dataclasses import dataclass +from email import message_from_bytes +from email.generator import BytesGenerator +from email.policy import compat32 +from installer.records import Hash, RecordEntry, parse_record_file +from packaging.metadata import RawMetadata, parse_email +from packaging.version import Version +from pathlib import Path + +from . import elf + +# Fixed member timestamp for files this tool adds, so two runs over the same +# inputs produce byte-identical wheels. +_ADDED_TIMESTAMP = (1980, 1, 1, 0, 0, 0) + + +def _reproducible_timestamp() -> tuple[int, int, int, int, int, int] | None: + """SOURCE_DATE_EPOCH as a zip timestamp, when a reproducible build asked for one. + + Zip records a local timestamp per member, so a wheel carries whatever clock + its builder had. Under SOURCE_DATE_EPOCH every member gets the same stamp -- + including ones copied through untouched -- or the rewrite would be + reproducible only for an input wheel that already was. + """ + epoch = os.environ.get("SOURCE_DATE_EPOCH") + if not epoch or not epoch.isdigit(): + return None + moment = time.gmtime(int(epoch)) + if moment.tm_year < 1980: # zip cannot record anything earlier + return _ADDED_TIMESTAMP + return ( + moment.tm_year, + moment.tm_mon, + moment.tm_mday, + moment.tm_hour, + moment.tm_min, + moment.tm_sec, + ) + + +class WheelError(Exception): + """The wheel is not shaped the way the packaging specs require.""" + + +@dataclass(frozen=True) +class Entry: + """One member of the wheel's zip archive.""" + + name: str + size: int + sha256: str + sha1: str + soname: str | None = None + needed: tuple[str, ...] = () + machine: str | None = None + # Enough to find a REUSE tag, kept so nothing re-opens the archive for it. + head: bytes = b"" + + @property + def is_elf(self) -> bool: + return self.machine is not None + + +@dataclass(frozen=True) +class WheelInfo: + path: Path + distribution: str + version: str + dist_info: str + entries: tuple[Entry, ...] + + +def record_hash(data: bytes) -> str: + """RECORD's hash field, as PEP 376 spells it.""" + return str(Hash.parse(f"sha256={_urlsafe_b64(data)}")) + + +def record_hash_from_sha256(digest: str) -> str: + """The same field, from a digest already computed.""" + encoded = ( + base64.urlsafe_b64encode(bytes.fromhex(digest)).rstrip(b"=").decode("ascii") + ) + return str(Hash.parse(f"sha256={encoded}")) + + +def _urlsafe_b64(data: bytes) -> str: + return ( + base64.urlsafe_b64encode(hashlib.sha256(data).digest()) + .rstrip(b"=") + .decode("ascii") + ) + + +def _dist_info(names: list[str]) -> str: + candidates = sorted( + { + name.split("/", 1)[0] + for name in names + if name.endswith(".dist-info/METADATA") + } + ) + if len(candidates) != 1: + raise WheelError( + f"expected exactly one .dist-info with METADATA, found {candidates}" + ) + return candidates[0] + + +def scan(wheel_path: Path, analyze_elf: bool = False) -> WheelInfo: + """Hash every member, and read the dynamic section of ELF members on request. + + `analyze_elf` is what the inventory needs and what verification does not, so + a consumer checking a downloaded wheel never runs a binary parser over it. + """ + with zipfile.ZipFile(wheel_path) as archive: + names = archive.namelist() + dist_info = _dist_info(names) + entries: list[Entry] = [] + for info in archive.infolist(): + if info.is_dir(): + continue + data = archive.read(info.filename) + soname = needed = machine = None + if analyze_elf and elf.is_elf(data[:4]): + dynamic = elf.read_dynamic(data) + soname, needed, machine = ( + dynamic.soname, + dynamic.needed, + dynamic.machine, + ) + entries.append( + Entry( + name=info.filename, + size=len(data), + sha256=hashlib.sha256(data).hexdigest(), + sha1=hashlib.sha1(data).hexdigest(), # noqa: S324 - SPDX 2.3 requires SHA1 + soname=soname, + needed=needed or (), + machine=machine, + head=data[:4096], + ) + ) + + stem = dist_info.removesuffix(".dist-info") + distribution, _, version = stem.partition("-") + if not version: + raise WheelError(f"cannot read distribution and version from {dist_info!r}") + return WheelInfo( + path=wheel_path, + distribution=distribution, + version=version, + dist_info=dist_info, + entries=tuple(entries), + ) + + +def read_metadata(wheel_path: Path, dist_info: str) -> tuple[bytes, RawMetadata]: + """Core metadata, parsed by packaging rather than as loose email headers.""" + with zipfile.ZipFile(wheel_path) as archive: + raw = archive.read(f"{dist_info}/METADATA") + return raw, parse_metadata(raw) + + +def parse_metadata(raw: bytes) -> RawMetadata: + parsed, _ = parse_email(raw) + return parsed + + +def add_license_files(raw_metadata: bytes, relative_paths: list[str]) -> bytes: + """Declare packaged license files, which PEP 639 requires of Metadata 2.4. + + Older metadata cannot express License-File, so the files still ship but go + undeclared; the caller reports that rather than emitting invalid metadata. + """ + metadata = parse_metadata(raw_metadata) + # Compare as a version: "2.10" sorts before "2.4" as a string. + if Version(metadata.get("metadata_version") or "0") < Version("2.4"): + return raw_metadata + + existing = set(metadata.get("license_files") or []) + additions = [path for path in relative_paths if path not in existing] + if not additions: + return raw_metadata + + # email owns RFC 822 header insertion, including keeping the payload (the + # long description) intact. + message = message_from_bytes(raw_metadata, policy=compat32) + for path in additions: + message.add_header("License-File", path) + + # maxheaderlen=0 disables RFC 822 folding. A folded value is still valid and + # packaging unfolds it, but no other tool wraps paths in METADATA, and a + # continuation line defeats anything reading it more simply. + buffer = io.BytesIO() + BytesGenerator(buffer, policy=compat32, maxheaderlen=0).flatten(message) + return buffer.getvalue() + + +def read_record(wheel_path: Path, dist_info: str) -> dict[str, tuple[str, str]]: + """Parse RECORD with installer, which owns the format's escaping rules.""" + with zipfile.ZipFile(wheel_path) as archive: + lines = archive.read(f"{dist_info}/RECORD").decode("utf-8").splitlines() + entries = (RecordEntry.from_elements(*row) for row in parse_record_file(lines)) + return { + entry.path: (str(entry.hash_) if entry.hash_ else "", str(entry.size or "")) + for entry in entries + } + + +def rewrite( + wheel_path: Path, + output_path: Path, + *, + additions: dict[str, bytes], + replacements: dict[str, bytes], + dist_info: str, +) -> None: + """Copy the wheel applying replacements and additions, then rebuild RECORD. + + RECORD is regenerated from what the new archive actually holds, so a wrong + digest cannot survive the step that writes it. + """ + record_name = f"{dist_info}/RECORD" + if record_name in additions or record_name in replacements: + raise WheelError("RECORD is regenerated here; do not pass it in") + + pinned = _reproducible_timestamp() + rows: list[tuple[str, str, str]] = [] + with ( + zipfile.ZipFile(wheel_path) as source, + zipfile.ZipFile(output_path, "w", zipfile.ZIP_DEFLATED) as target, + ): + for info in source.infolist(): + if info.is_dir() or info.filename == record_name: + continue + data = replacements.get(info.filename) + if data is None: + data = source.read(info.filename) + member = zipfile.ZipInfo(info.filename, date_time=pinned or info.date_time) + member.external_attr = info.external_attr + member.compress_type = zipfile.ZIP_DEFLATED + target.writestr(member, data) + rows.append((info.filename, record_hash(data), str(len(data)))) + + for name in sorted(additions): + data = additions[name] + member = zipfile.ZipInfo(name, date_time=pinned or _ADDED_TIMESTAMP) + member.external_attr = 0o644 << 16 + member.compress_type = zipfile.ZIP_DEFLATED + target.writestr(member, data) + rows.append((name, record_hash(data), str(len(data)))) + + buffer = io.StringIO() + writer = csv.writer(buffer, lineterminator="\n") + for row in sorted(rows): + writer.writerow(row) + writer.writerow((record_name, "", "")) + member = zipfile.ZipInfo(record_name, date_time=pinned or _ADDED_TIMESTAMP) + member.external_attr = 0o644 << 16 + member.compress_type = zipfile.ZIP_DEFLATED + target.writestr(member, buffer.getvalue()) diff --git a/src/core/python/CMakeLists.txt b/src/core/python/CMakeLists.txt index 1685616ef6..49fca70e8d 100644 --- a/src/core/python/CMakeLists.txt +++ b/src/core/python/CMakeLists.txt @@ -43,6 +43,10 @@ add_custom_target(python_package ALL COMMAND ${CMAKE_COMMAND} -E copy "${CMAKE_CURRENT_SOURCE_DIR}/MANIFEST.in" "${CMAKE_BINARY_DIR}/python_package/$/" COMMAND ${CMAKE_COMMAND} -E copy "${CMAKE_CURRENT_SOURCE_DIR}/setup.py" "${CMAKE_BINARY_DIR}/python_package/$/" COMMAND ${CMAKE_COMMAND} -E copy "${CMAKE_SOURCE_DIR}/README.md" "${CMAKE_BINARY_DIR}/python_package/$/README.md" + # project.license-files in pyproject.toml.in names these; without them the + # wheel redistributes the project under a license it does not carry. + COMMAND ${CMAKE_COMMAND} -E copy "${CMAKE_SOURCE_DIR}/LICENSE.md" "${CMAKE_BINARY_DIR}/python_package/$/LICENSE.md" + COMMAND ${CMAKE_COMMAND} -E copy_directory "${CMAKE_SOURCE_DIR}/LICENSES" "${CMAKE_BINARY_DIR}/python_package/$/LICENSES" COMMAND ${CMAKE_COMMAND} -E copy "${CMAKE_CURRENT_SOURCE_DIR}/requirements.txt" "${CMAKE_BINARY_DIR}/python_package/$/" COMMAND ${CMAKE_COMMAND} -E copy "${CMAKE_CURRENT_SOURCE_DIR}/requirements-dev.txt" "${CMAKE_BINARY_DIR}/python_package/$/" COMMAND ${CMAKE_COMMAND} -E copy "${CMAKE_CURRENT_SOURCE_DIR}/requirements-ui.txt" "${CMAKE_BINARY_DIR}/python_package/$/" diff --git a/src/core/python/pyproject.toml.in b/src/core/python/pyproject.toml.in index 327a0b66dd..681d408787 100644 --- a/src/core/python/pyproject.toml.in +++ b/src/core/python/pyproject.toml.in @@ -9,7 +9,10 @@ # See: numpy.org/devdocs/dev/depending_on_numpy.html [build-system] -requires = ["setuptools>=64", "wheel", "numpy>=2.0"] +# setuptools>=77 for PEP 639: the SPDX `license` string and `license-files` +# below are what put LICENSE.md and LICENSES/ under .dist-info/licenses/ and +# name them in METADATA. The SBOM collector appends third-party texts there. +requires = ["setuptools>=77", "wheel", "numpy>=2.0"] build-backend = "setuptools.build_meta" [project] @@ -19,6 +22,8 @@ description = "Isaac Teleop - Teleoperation with Device I/O" readme = "README.md" requires-python = ">=3.11" license = "Apache-2.0" +# Staged beside this file by the python_package target. +license-files = ["LICENSE.md", "LICENSES/*.txt"] authors = [ {name = "NVIDIA"} ] diff --git a/tests/python/CMakeLists.txt b/tests/python/CMakeLists.txt index 84d64a2e60..743c0ad493 100644 --- a/tests/python/CMakeLists.txt +++ b/tests/python/CMakeLists.txt @@ -14,6 +14,9 @@ if(BUILD_PYTHON_BINDINGS) add_subdirectory(compat) endif() +# Tooling tests: synthetic fixtures, so no built package is needed. +add_subdirectory(sbom) + if(BUILD_EXAMPLE_TELEOP_ROS2) add_subdirectory(examples/teleop_ros2) endif() diff --git a/tests/python/sbom/CMakeLists.txt b/tests/python/sbom/CMakeLists.txt new file mode 100644 index 0000000000..4403e6fc40 --- /dev/null +++ b/tests/python/sbom/CMakeLists.txt @@ -0,0 +1,31 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# ============================================================================== +# SBOM collector tests +# ============================================================================== +# These drive synthetic wheels and build trees, so they need no built package +# and no configured dependency tree -- only the SPDX data the build fetched. + +file(GLOB TEST_FILES + RELATIVE "${CMAKE_CURRENT_SOURCE_DIR}" + "${CMAKE_CURRENT_SOURCE_DIR}/test_*.py" +) + +foreach(test_file ${TEST_FILES}) + get_filename_component(test_name "${test_file}" NAME_WE) + + add_test( + NAME "sbom_${test_name}" + COMMAND uv run --python ${ISAAC_TELEOP_PYTHON_VERSION} --extra dev pytest -v --tb=short "${test_file}" + WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}" + ) + + # The SPDX license list deps/third_party fetched for this build, so the tests + # read the same reference texts the collector will and never reach the + # network; and this build tree, so the file API reader is exercised against + # a codemodel CMake actually wrote. + set_tests_properties("sbom_${test_name}" PROPERTIES + ENVIRONMENT "ISAACTELEOP_SBOM_LICENSE_DATA=${SPDX_LICENSE_DATA_JSON_DIR};ISAACTELEOP_SBOM_BUILD_DIR=${CMAKE_BINARY_DIR}" + ) +endforeach() diff --git a/tests/python/sbom/conftest.py b/tests/python/sbom/conftest.py new file mode 100644 index 0000000000..8a67ddef8d --- /dev/null +++ b/tests/python/sbom/conftest.py @@ -0,0 +1,137 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Fixtures for the SBOM collector tests.""" + +from __future__ import annotations + +import importlib.util +import json +import os +from pathlib import Path + +import pytest +import synth +from sbom import build as build_module +from sbom import licensing + + +def _load(name: str, path: Path): + """Import a sibling helper by path. + + Not via `sys.path`: this directory is named `sbom`, so putting its parent on + the path would shadow the installed collector package with a namespace one. + """ + spec = importlib.util.spec_from_file_location(name, path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +repo_root = _load( + "repo_paths", Path(__file__).resolve().parents[1] / "repo_paths.py" +).repo_root + +LICENSE_DATA_ENV = "ISAACTELEOP_SBOM_LICENSE_DATA" +BUILD_DIR_ENV = "ISAACTELEOP_SBOM_BUILD_DIR" + + +@pytest.fixture(scope="session") +def license_data(): + """The SPDX license list deps/third_party fetched into this build's _deps. + + CTest points the environment variable at it. Outside that, fall back to a + default build directory; if neither exists the license tests skip rather + than run against data that is not what the build pinned. + """ + candidates = [] + if os.environ.get(LICENSE_DATA_ENV): + candidates.append(Path(os.environ[LICENSE_DATA_ENV])) + candidates.append( + repo_root() / "build" / "_deps" / "license-list-data-src" / "json" + ) + + for candidate in candidates: + try: + licensing.load_corpus(candidate) + return candidate + except licensing.CorpusError: + continue + + pytest.skip( + "SPDX license list data not found; configure and build so deps/third_party " + "materializes it, or set " + LICENSE_DATA_ENV + ) + + +@pytest.fixture(autouse=True) +def _license_data_loaded(license_data): + """Load the reference texts before each test. + + Reloading matters: a `build()` call points the collector at its own build + tree, which for the fixtures holds only a handful of licenses. Without this + the next test would inherit that subset. + """ + licensing.load_corpus(license_data) + synth.set_license_data(license_data) + + +@pytest.fixture(scope="session") +def checkout(): + """This repository, as the collector would be pointed at it.""" + return repo_root() + + +@pytest.fixture(scope="session") +def live_build_dir(): + """This repository's own configured build tree, when there is one. + + The synthetic workspaces are written to a model of what CMake emits. This is + the anchor to what it actually emits, so a change to the codemodel's shape + fails here instead of passing everywhere. + """ + candidates = [] + if os.environ.get(BUILD_DIR_ENV): + candidates.append(Path(os.environ[BUILD_DIR_ENV])) + candidates.append(repo_root() / "build") + + for candidate in candidates: + if sorted( + (candidate / ".cmake" / "api" / "v1" / "reply").glob("codemodel-v2-*.json") + ): + return candidate + + pytest.skip("no configured build tree to read; configure this project first") + + +@pytest.fixture +def workspace(tmp_path): + """A generated project, its build tree, and the wheel that build produced.""" + space = synth.create(tmp_path / "project") + synth.write_wheel(space.wheel, synth.wheel_payload(space)) + return space + + +@pytest.fixture +def built(workspace): + """The workspace after the collector has packaged evidence into the wheel.""" + out_dir = workspace.root / "sbom" + manifest = build_module.build( + workspace.root, workspace.build, workspace.wheel, out_dir + ) + manifest_path = out_dir / "manifest.json" + manifest_path.write_text( + json.dumps(manifest, indent=2, sort_keys=True) + "\n", encoding="utf-8" + ) + stem = synth.WHEEL_NAME.removesuffix(".whl") + return { + "workspace": workspace, + "root": workspace.root, + "wheel": workspace.wheel, + "out_dir": out_dir, + "manifest": manifest, + "manifest_path": manifest_path, + "evidence_path": out_dir / f"{stem}.build-evidence.json", + "spdx_path": out_dir / f"{stem}.spdx.json", + "report_path": out_dir / f"{stem}.licenses.json", + } diff --git a/tests/python/sbom/pyproject.toml b/tests/python/sbom/pyproject.toml new file mode 100644 index 0000000000..0bf1586bc1 --- /dev/null +++ b/tests/python/sbom/pyproject.toml @@ -0,0 +1,23 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +[project] +name = "isaaccapture-sbom-tests" +version = "0.0.0" # Internal tests - not versioned +description = "Tests for the wheel SBOM and license evidence collector" +requires-python = ">=3.11,<3.14" + +[project.optional-dependencies] +# The collector itself, from the source tree: its pins live in its own +# pyproject, so there is one place to bump them. +dev = ["isaaccapture-sbom[dev]"] + +[tool.uv.sources] +isaaccapture-sbom = { path = "../../../scripts/sbom", editable = true } + +[tool.pytest.ini_options] +pythonpath = ["."] +python_files = ["test_*.py"] +python_classes = ["Test*"] +python_functions = ["test_*"] +norecursedirs = [".git", ".venv", "build", "dist", "*.egg", "__pycache__"] diff --git a/tests/python/sbom/synth.py b/tests/python/sbom/synth.py new file mode 100644 index 0000000000..fd1e96a417 --- /dev/null +++ b/tests/python/sbom/synth.py @@ -0,0 +1,513 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""A synthetic project, build tree and wheel for the SBOM collector's tests. + +The fixture has to be a build tree: a file API reply, fetched source trees, an +SDK tarball, a staged package directory. Generated rather than built, because the +gates need inputs a working build cannot produce -- a member from nowhere, a +component with no license text, a tampered wheel -- and because the suite must +run with no compiler, no network and no configured CMake. + +Keep every reader anchored to something real as well, or a fixture the collector +agrees with proves nothing: the codemodel reader against this repo's own build +tree, the ELF reader against a system library, the matcher against the fetched +SPDX corpus. +""" + +from __future__ import annotations + +import base64 +import csv +import gzip +import hashlib +import io +import json +import struct +import tarfile +import zipfile +from pathlib import Path + +CONFIG = "Release" +WHEEL_NAME = "isaaccapture-0.4.0-cp312-cp312-manylinux_2_35_x86_64.whl" +DIST_INFO = "isaaccapture-0.4.0.dist-info" +SDK_ARCHIVE = "deps/vendor/demosdk-1.2.0-linux-amd64.tar.gz" + +# ============================================================================== +# ELF +# ============================================================================== + +_PT_LOAD = 1 +_PT_DYNAMIC = 2 +_DT_NULL = 0 +_DT_NEEDED = 1 +_DT_STRTAB = 5 +_DT_STRSZ = 10 +_DT_SONAME = 14 +_EM_X86_64 = 62 +_EHDR_SIZE = 64 +_PHDR_SIZE = 56 +_BASE_VADDR = 0x400000 + + +def elf_shared_object( + soname: str, needed: tuple[str, ...] = (), padding: bytes = b"" +) -> bytes: + """A 64-bit little-endian ELF carrying just a readable dynamic segment.""" + strings = [b"\0"] # index 0 is the empty string, per the ELF string-table format + offsets: dict[str, int] = {} + cursor = 1 + for value in (soname, *needed): + if value in offsets: + continue + offsets[value] = cursor + encoded = value.encode("utf-8") + b"\0" + strings.append(encoded) + cursor += len(encoded) + strtab = b"".join(strings) + + entries = [(_DT_SONAME, offsets[soname])] + entries += [(_DT_NEEDED, offsets[name]) for name in needed] + + header_size = _EHDR_SIZE + 2 * _PHDR_SIZE + dynamic_offset = header_size + dynamic_size = (len(entries) + 3) * 16 + strtab_offset = dynamic_offset + dynamic_size + total = strtab_offset + len(strtab) + len(padding) + + entries.append((_DT_STRTAB, _BASE_VADDR + strtab_offset)) + entries.append((_DT_STRSZ, len(strtab))) + entries.append((_DT_NULL, 0)) + + image = bytearray(total) + image[0:4] = b"\x7fELF" + image[4] = 2 # ELFCLASS64 + image[5] = 1 # ELFDATA2LSB + image[6] = 1 # EV_CURRENT + struct.pack_into(" None: + base = _EHDR_SIZE + index * _PHDR_SIZE + struct.pack_into(" None: + """Point the fixtures at the SPDX json/ directory the build materialized.""" + global _LICENSE_DATA + _LICENSE_DATA = path + + +def license_data() -> Path: + if _LICENSE_DATA is None: + raise RuntimeError("set_license_data() has not been called") + return _LICENSE_DATA + + +def spdx_text(license_id: str) -> str: + """The canonical text of a license, from the data the collector reads.""" + payload = json.loads( + (license_data() / "details" / f"{license_id}.json").read_text(encoding="utf-8") + ) + return payload["licenseText"] + + +PROPRIETARY_LICENSE = """\ +DEMO CORP SOFTWARE LICENSE AGREEMENT + +This agreement governs your use of the Demo SDK. You may redistribute the +runtime libraries with your application provided this notice accompanies them. +The software is provided as is, without warranty of any kind. +""" + + +# ============================================================================== +# Workspace +# ============================================================================== + +_DEPS = { + # key: (remote, license file name, SPDX id whose canonical text it ships) + "alpha": ("https://github.com/example-org/alpha.git", "LICENSE", "MIT"), + "beta": ("https://github.com/example-org/beta.git", "LICENSE", "Apache-2.0"), + "gamma": ("https://github.com/example-org/gamma.git", "LICENSE", "Zlib"), + "delta": ( + "https://github.com/example-org/delta.git", + "BSD-LICENSE", + "BSD-3-Clause", + ), + "omega": ("https://github.com/example-org/omega.git", "LICENSE.txt", "BSL-1.0"), + "vendorpy": ( + "https://github.com/example-org/vendorpy.git", + "LICENSE", + "Apache-2.0", + ), +} + +_CACHE = """\ +# This is the CMakeCache file. +CMAKE_BUILD_TYPE:STRING=Release +CMAKE_SYSTEM_PROCESSOR:STRING=x86_64 +CMAKE_CXX_COMPILER:FILEPATH=/usr/bin/c++ +ISAAC_TELEOP_PYTHON_VERSION:STRING=3.12 +BUILD_VIZ:BOOL=ON +BUILD_PYTHON_BINDINGS:BOOL=ON +""" + +_AUTHORED_INIT = """\ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. +# SPDX-License-Identifier: Apache-2.0 + +__version__ = "0.4.0" +""" +_GENERATED = "# generated at configure time\nTRACKERS = ()\n" +_VENDOR_MODULE = """\ +# SPDX-FileCopyrightText: Copyright (c) 2026 Example Org +# SPDX-License-Identifier: Apache-2.0 + +def retarget(): + return None +""" +# Checked in rather than fetched, and says whose it is the way upstream files +# usually do: a plain copyright line, no REUSE tag. +_VENDORED_HEADER = """\ +// Copyright 2024, Upstream Widgets Ltd. +// SPDX-License-Identifier: BSL-1.0 +#pragma once +""" + +_PROJECT_PYPROJECT = """\ +[project] +name = "isaaccapture" +version = "0.4.0" +authors = [{ name = "NVIDIA" }] +""" + +_VENDOR_ASSET_SOURCE = ( + '\n' +) +_VENDOR_ASSET_STAGED = '\n' + + +class Workspace: + """Paths into a generated project, and the wheel its build would produce.""" + + def __init__(self, root: Path) -> None: + self.root = root + self.build = root / "build" + self.staged = self.build / "python_package" / CONFIG + self.wheel = root / "dist" / WHEEL_NAME + + +def _write(path: Path, data: bytes | str) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + if isinstance(data, str): + path.write_text(data, encoding="utf-8") + else: + path.write_bytes(data) + + +def _stage_license_data(target: Path) -> None: + """Mirror what deps/third_party leaves in _deps, with just the licenses used here.""" + source = license_data() + _write( + target / "licenses.json", (source / "licenses.json").read_text(encoding="utf-8") + ) + for license_id in FIXTURE_LICENSES: + detail = source / "details" / f"{license_id}.json" + if detail.is_file(): + _write(target / "details" / detail.name, detail.read_text(encoding="utf-8")) + + +def _gitclone_stub(deps: Path, key: str, remote: str) -> None: + tmp = deps / f"{key}-subbuild" / f"{key}-populate-prefix" / "tmp" + tmp.mkdir(parents=True, exist_ok=True) + _write( + tmp / f"{key}-populate-gitclone.cmake", + " clone --no-checkout --depth 1 --no-single-branch " + f'--config "advice.detachedHead=false" "{remote}" "{key}-src"\n' + ' checkout "v1.2.3" --\n', + ) + + +def _sdk_tarball(path: Path) -> dict[str, bytes]: + """An SDK archive whose libraries are redistributed, one of them patched.""" + members = { + "LICENSE.txt": PROPRIETARY_LICENSE.encode("utf-8"), + "VERSION": b"1.2.0\n", + "libdemo.so": elf_shared_object("libdemo.so", ("libc.so.6",)), + "libextra.so": elf_shared_object("libextra.so", ("libssl.so.3", "libc.so.6")), + "include/demo.h": b"#pragma once\n", + } + # Fixed mtimes, on the members and on the gzip stream: "w:gz" stamps the + # current time into both, and two archives written a second apart are not + # the same input. + path.parent.mkdir(parents=True, exist_ok=True) + with ( + gzip.GzipFile(filename=path, mode="wb", mtime=0) as compressed, + tarfile.open(fileobj=compressed, mode="w") as archive, + ): + for name, data in sorted(members.items()): + info = tarfile.TarInfo(f"./{name}") + info.size = len(data) + info.mtime = 0 + archive.addfile(info, io.BytesIO(data)) + return members + + +def create(root: Path) -> Workspace: + """Generate the project, its build tree, and the staged package.""" + workspace = Workspace(root) + build = workspace.build + deps = build / "_deps" + + _write(root / "VERSION", "0.4.x\n") + _write(root / "CMakeLists.txt", "cmake_minimum_required(VERSION 3.24)\n") + _write(root / "LICENSE.md", spdx_text("Apache-2.0")) + for pool_id in ("Apache-2.0", "BSL-1.0"): + _write(root / "LICENSES" / f"{pool_id}.txt", spdx_text(pool_id)) + _write(root / "src/python/isaaccapture/__init__.py", _AUTHORED_INIT) + _write(root / "src/vendor/upstream_helper.h", _VENDORED_HEADER) + _write(root / "pyproject.toml", _PROJECT_PYPROJECT) + _write(build / "CMakeCache.txt", _CACHE) + + _stage_license_data(deps / "license-list-data-src" / "json") + + for key, (remote, license_name, license_id) in _DEPS.items(): + source = deps / f"{key}-src" + _write(source / license_name, spdx_text(license_id)) + _write(source / f"{key}.cpp", f"// {key}\n") + _gitclone_stub(deps, key, remote) + + # vendorpy ships Python that the build stages into the wheel. + _write(deps / "vendorpy-src/pkg/mod.py", _VENDOR_MODULE) + _write(deps / "vendorpy-src/pkg/assets/hand.xml", _VENDOR_ASSET_SOURCE) + + sdk_members = _sdk_tarball(root / SDK_ARCHIVE) + + # --- build outputs ------------------------------------------------------- + extension = "isaaccapture/_ext.cpython-312-x86_64-linux-gnu.so" + extension_bytes = elf_shared_object( + "_ext.cpython-312-x86_64-linux-gnu.so", ("libstdc++.so.6", "libc.so.6") + ) + gamma_bytes = elf_shared_object("libgamma_renamed.so", ("libc.so.6",)) + _write(workspace.staged / extension, extension_bytes) + _write(build / "lib/libgamma_renamed.so", gamma_bytes) + _write(build / "lib/libdelta.a", b"!\ndelta\n") + _write(build / "generated/_generated.py", _GENERATED) + + # Staged tree: what the wheel is built from. + _write(workspace.staged / "isaaccapture/__init__.py", _AUTHORED_INIT) + _write(workspace.staged / "isaaccapture/_generated.py", _GENERATED) + _write(workspace.staged / "isaaccapture/viz/libgamma_renamed.so", gamma_bytes) + _write(workspace.staged / "isaaccapture/viz/GAMMA_LICENSE", spdx_text("Zlib")) + _write( + workspace.staged / "isaaccapture/sdk/native/libdemo.so", + sdk_members["libdemo.so"], + ) + # patchelf drops a NEEDED entry, so the shipped bytes differ from the archive's. + _write( + workspace.staged / "isaaccapture/sdk/native/libextra.so", + elf_shared_object("libextra.so", ("libc.so.6",)), + ) + _write(workspace.staged / "vendorpy/mod.py", _VENDOR_MODULE) + _write(workspace.staged / "vendorpy/assets/hand.xml", _VENDOR_ASSET_STAGED) + + _file_api_reply( + root, build, (workspace.staged / extension).relative_to(build).as_posix() + ) + _write(build / "tests/unit_tests", b"\x7fELF-not-really") + return workspace + + +def _file_api_reply(root: Path, build: Path, extension: str) -> None: + """Write the codemodel a `cmake_file_api()` query leaves behind. + + The collector reads CMake's file API rather than scraping generator output, + so the fixture has to speak the same thing a real configure produces. + """ + targets = [ + { + "name": "ext", + "type": "MODULE_LIBRARY", + # The project's own source, compiled against beta's headers, with + # alpha's source compiled straight in. + "artifacts": [extension], + "sources": ["src/cpp/module.cpp", "build/_deps/alpha-src/alpha.cpp"], + "includes": [ + "build/_deps/beta-src/include", + "build/_deps/alpha-src", + "src/vendor", + ], + "dependencies": [], + }, + { + "name": "gamma", + "type": "SHARED_LIBRARY", + "artifacts": ["lib/libgamma_renamed.so"], + "sources": ["build/_deps/gamma-src/gamma.cpp"], + "includes": ["build/_deps/gamma-src"], + "dependencies": ["delta"], + }, + { + "name": "delta", + "type": "STATIC_LIBRARY", + "artifacts": ["lib/libdelta.a"], + "sources": ["build/_deps/delta-src/delta.cpp"], + "includes": ["build/_deps/delta-src"], + "dependencies": [], + }, + { + # Tests link omega; nothing they produce reaches a wheel. + "name": "unit_tests", + "type": "EXECUTABLE", + "artifacts": ["tests/unit_tests"], + "sources": ["tests/cpp/test_main.cpp"], + "includes": ["build/_deps/omega-src/include"], + "dependencies": [], + }, + ] + + reply = build / ".cmake" / "api" / "v1" / "reply" + index = [] + for spec in targets: + name = spec["name"] + payload = { + "name": name, + "type": spec["type"], + "artifacts": [{"path": item} for item in spec["artifacts"]], + "sources": [{"path": item} for item in spec["sources"]], + "compileGroups": [ + {"includes": [{"path": item} for item in spec["includes"]]} + ], + "dependencies": [{"id": dep} for dep in spec["dependencies"]], + } + _write(reply / f"target-{name}-Release.json", json.dumps(payload, indent=1)) + index.append( + {"id": name, "name": name, "jsonFile": f"target-{name}-Release.json"} + ) + + codemodel = { + "kind": "codemodel", + "version": {"major": 2, "minor": 6}, + "paths": {"source": str(root), "build": str(build)}, + "configurations": [{"name": CONFIG, "targets": index}], + } + _write(reply / "codemodel-v2-fixture.json", json.dumps(codemodel, indent=1)) + + +# ============================================================================== +# Wheel +# ============================================================================== + +METADATA = "\n".join( + [ + "Metadata-Version: 2.4", + "Name: isaaccapture", + "Version: 0.4.0", + "Summary: Isaac Capture", + "License-Expression: Apache-2.0", + "License-File: LICENSE.md", + "Requires-Python: >=3.11", + "Requires-Dist: numpy>=1.23.0", + "Requires-Dist: pyyaml>=6.0.3", + "", + "Isaac Capture long description.", + "", + ] +) + + +def wheel_payload(workspace: Workspace) -> dict[str, bytes]: + """The wheel setuptools would produce from the staged tree.""" + payload: dict[str, bytes] = {} + for path in sorted(workspace.staged.rglob("*")): + if path.is_file(): + payload[path.relative_to(workspace.staged).as_posix()] = path.read_bytes() + + payload[f"{DIST_INFO}/METADATA"] = METADATA.encode("utf-8") + payload[f"{DIST_INFO}/WHEEL"] = ( + b"Wheel-Version: 1.0\nGenerator: setuptools\nRoot-Is-Purelib: false\n" + ) + payload[f"{DIST_INFO}/licenses/LICENSE.md"] = ( + workspace.root / "LICENSE.md" + ).read_bytes() + payload[f"{DIST_INFO}/top_level.txt"] = b"isaaccapture\nvendorpy\n" + return payload + + +def _record_hash(data: bytes) -> str: + digest = hashlib.sha256(data).digest() + return "sha256=" + base64.urlsafe_b64encode(digest).rstrip(b"=").decode("ascii") + + +def write_wheel(path: Path, payload: dict[str, bytes]) -> Path: + """Write a wheel with a correct RECORD, the way a real build leaves one.""" + dist_info = next( + name.split("/", 1)[0] + for name in payload + if name.endswith(".dist-info/METADATA") + ) + record_name = f"{dist_info}/RECORD" + + buffer = io.StringIO() + writer = csv.writer(buffer, lineterminator="\n") + for name in sorted(payload): + writer.writerow((name, _record_hash(payload[name]), str(len(payload[name])))) + writer.writerow((record_name, "", "")) + + # Fixed stamps: writestr would otherwise record wall-clock, and two wheels + # written either side of a second boundary are not the same input. + path.parent.mkdir(parents=True, exist_ok=True) + with zipfile.ZipFile(path, "w", zipfile.ZIP_DEFLATED) as archive: + for name in [*sorted(payload), record_name]: + data = ( + payload[name] if name in payload else buffer.getvalue().encode("utf-8") + ) + member = zipfile.ZipInfo(name, date_time=(1980, 1, 1, 0, 0, 0)) + member.external_attr = 0o644 << 16 + member.compress_type = zipfile.ZIP_DEFLATED + archive.writestr(member, data) + return path diff --git a/tests/python/sbom/test_discovery.py b/tests/python/sbom/test_discovery.py new file mode 100644 index 0000000000..4dbab85130 --- /dev/null +++ b/tests/python/sbom/test_discovery.py @@ -0,0 +1,190 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""What the build left behind is the only dependency declaration there is.""" + +from __future__ import annotations + +import pytest +import synth +from sbom import discovery + + +def test_source_trees_are_found_without_being_listed(workspace): + components = discovery.discover_source_trees(workspace.build / "_deps") + + # license-list-data is the SPDX reference data deps/third_party fetches. It is + # fetched the same way as everything else, so it is discovered the same way. + assert set(components) == { + "alpha", + "beta", + "delta", + "gamma", + "license-list-data", + "omega", + "vendorpy", + } + + +def test_identity_comes_off_the_remote_url(workspace): + alpha = discovery.discover_source_trees(workspace.build / "_deps")["alpha"] + + assert alpha.supplier == "Organization: example-org" + assert alpha.homepage == "https://github.com/example-org/alpha" + assert alpha.purl.startswith("pkg:github/example-org/alpha") + assert alpha.version == "v1.2.3", "the declared constraint stands in for a commit" + + +def test_no_checkout_on_the_clone_line_is_not_a_checkout(workspace): + """`git clone --no-checkout ` must not be read as `git checkout `.""" + beta = discovery.discover_source_trees(workspace.build / "_deps")["beta"] + + assert not beta.version.startswith("http") + + +def test_licenses_come_from_the_trees_own_files(workspace): + components = discovery.discover_source_trees(workspace.build / "_deps") + + assert components["alpha"].license_concluded == "MIT" + assert components["delta"].license_concluded == "BSD-3-Clause" + assert components["gamma"].license_concluded == "Zlib" + + +def test_link_graph_finds_static_and_header_only_contributions(workspace): + graph = discovery.BuildGraph(workspace.build) + extension = graph.by_name("_ext.cpython-312-x86_64-linux-gnu.so") + + assert len(extension) == 1 + contributions = graph.contributions(extension[0]) + + assert contributions["alpha"] == {"compiled-source", "header-include"} + # beta ships headers only; an include path is the only trace it leaves. + assert contributions["beta"] == {"header-include"} + + +def test_link_graph_recurses_through_static_archives(workspace): + """delta reaches the wheel only through libdelta.a inside libgamma_renamed.so.""" + graph = discovery.BuildGraph(workspace.build) + gamma = graph.by_name("libgamma_renamed.so")[0] + + contributions = graph.contributions(gamma) + + assert "gamma" in contributions + assert "delta" in contributions + + +def test_first_party_is_decided_by_whose_sources_were_compiled(workspace): + graph = discovery.BuildGraph(workspace.build) + repo = workspace.root.resolve() + + extension = graph.by_name("_ext.cpython-312-x86_64-linux-gnu.so")[0] + gamma = graph.by_name("libgamma_renamed.so")[0] + + assert graph.first_party(extension, repo) + assert not graph.first_party(gamma, repo) + + +def test_a_test_only_dependency_reaches_no_shipped_artifact(workspace): + graph = discovery.BuildGraph(workspace.build) + extension = graph.by_name("_ext.cpython-312-x86_64-linux-gnu.so")[0] + gamma = graph.by_name("libgamma_renamed.so")[0] + + assert "omega" not in graph.contributions(extension) + assert "omega" not in graph.contributions(gamma) + assert "omega" in graph.contributions(graph.by_name("unit_tests")[0]) + + +def test_archive_members_are_indexed_by_content(workspace): + index = discovery.discover_archives(workspace.root) + + assert synth.SDK_ARCHIVE in index.archives + demo = index.by_name["libdemo.so"] + assert len(demo) == 1 + assert index.by_hash[demo[0].sha256].container == synth.SDK_ARCHIVE + + +def test_archive_version_and_license_come_from_the_archive(workspace): + index = discovery.discover_archives(workspace.root) + record = index.archives[synth.SDK_ARCHIVE] + + assert record["version_text"] == "1.2.0" + assert [member.path for member, _ in record["license_members"]] == ["LICENSE.txt"] + + +def test_a_build_tree_without_a_file_api_reply_says_so(workspace): + """The query lives in CMakeLists, so a missing reply means the tree was + configured by a CMake too old to honour it — say that, don't guess.""" + import shutil + + shutil.rmtree(workspace.build / ".cmake") + + with pytest.raises(discovery.FileApiError, match="cmake_file_api"): + discovery.BuildGraph(workspace.build, synth.CONFIG) + + +def test_the_graph_is_read_from_the_codemodel_not_from_generator_output(workspace): + """link.txt is a Makefile-generator artifact; Ninja writes none. Reading the + file API is what makes the collector work under either.""" + assert not list(workspace.build.rglob("link.txt")) + assert not (workspace.build / "compile_commands.json").exists() + + graph = discovery.BuildGraph(workspace.build, synth.CONFIG) + + assert graph.by_name("_ext.cpython-312-x86_64-linux-gnu.so") + assert graph.by_name("libgamma_renamed.so") + + +def test_third_party_checked_in_here_is_still_third_party(workspace): + """Not everything third-party arrives through FetchContent. A file checked + into the repository that someone else wrote says so in its copyright line, + and that is what separates it from the project's own code -- not the + directory it sits in.""" + from sbom import evidence as evidence_module + + space = evidence_module.discover(workspace.root, workspace.build) + vendored = { + key: component + for key, component in space.components.items() + if component.kind == "vendored-source" + } + + assert [c.name for c in vendored.values()] == ["Upstream Widgets Ltd"] + component = next(iter(vendored.values())) + assert component.license_concluded == "BSL-1.0" + # Its text comes from this repository's REUSE pool, which is what the pool + # is for: the file states an identifier and carries no text of its own. + assert [item.path for item in component.evidence] == ["LICENSES/BSL-1.0.txt"] + + +def test_a_vendored_header_is_credited_to_the_artifact_that_includes_it(workspace): + from sbom import evidence as evidence_module + + space = evidence_module.discover(workspace.root, workspace.build) + artifact = space.graph.by_name("_ext.cpython-312-x86_64-linux-gnu.so")[0] + + contributions = space.graph.contributions(artifact, space.ownership) + + assert "vendored:upstream-widgets-ltd" in contributions + assert contributions["vendored:upstream-widgets-ltd"] == {"header-include"} + + +def test_the_codemodel_reader_works_on_a_codemodel_cmake_wrote( + live_build_dir, checkout +): + """Reads this repository's own build tree, not a synthesized one. + + Deliberately makes no claim about which dependencies are present -- only + about the shape of what the reader gets back, so a bump cannot break it. + """ + graph = discovery.BuildGraph(live_build_dir) + + assert graph.artifacts, "a configured build declares at least one artifact" + assert any( + graph.first_party(artifact, checkout) for artifact in graph.artifacts.values() + ), "this project compiles some of its own sources" + + for output, artifact in graph.artifacts.items(): + assert output.is_absolute() + assert graph.by_name(output.name), "every artifact is findable by name" + for how in graph.contributions(artifact).values(): + assert how <= {"compiled-source", "header-include"} diff --git a/tests/python/sbom/test_elf.py b/tests/python/sbom/test_elf.py new file mode 100644 index 0000000000..412021a95d --- /dev/null +++ b/tests/python/sbom/test_elf.py @@ -0,0 +1,72 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""The ELF reader is how bundled libraries are told apart from external ones.""" + +from __future__ import annotations + +import pytest +import synth +from sbom import elf + + +def test_reads_soname_and_needed(): + image = synth.elf_shared_object("libcloudxr.so", ("libPoco.so", "libcudart.so.12")) + info = elf.read_dynamic(image) + + assert info.soname == "libcloudxr.so" + assert info.needed == ("libPoco.so", "libcudart.so.12") + assert info.machine == "x86_64" + + +def test_reads_a_library_with_no_dependencies(): + info = elf.read_dynamic(synth.elf_shared_object("libisaacteleop_mujoco.so")) + + assert info.soname == "libisaacteleop_mujoco.so" + assert info.needed == () + + +def test_rejects_non_elf_payloads(): + assert not elf.is_elf(b"# not an elf\n") + with pytest.raises(elf.NotAnElf): + elf.read_dynamic(b"PK\x03\x04 zip, not elf") + + +def test_reads_a_real_system_library(): + """Guards the reader against the synthesizer agreeing with itself.""" + import ctypes.util + + name = ctypes.util.find_library("c") + if name is None: + pytest.skip("no system libc to read") + for candidate in ( + "/lib/x86_64-linux-gnu", + "/lib/aarch64-linux-gnu", + "/usr/lib", + "/lib", + ): + path = f"{candidate}/{name}" + try: + data = open(path, "rb").read() # noqa: SIM115 + except OSError: + continue + if elf.is_elf(data): + assert elf.read_dynamic(data).soname == name + return + pytest.skip("could not locate the system libc on disk") + + +def test_verification_does_not_parse_binaries(built): + """A consumer checking a downloaded wheel should not run a binary parser + over it; scanning only reads the dynamic section when asked to.""" + from sbom import wheelfile + + plain = wheelfile.scan(built["wheel"]) + analyzed = wheelfile.scan(built["wheel"], analyze_elf=True) + + assert not any(entry.is_elf for entry in plain.entries) + assert any(entry.is_elf for entry in analyzed.entries) + # The digests verification relies on are identical either way. + assert {e.name: e.sha256 for e in plain.entries} == { + e.name: e.sha256 for e in analyzed.entries + } diff --git a/tests/python/sbom/test_gates.py b/tests/python/sbom/test_gates.py new file mode 100644 index 0000000000..9bfdd36c5c --- /dev/null +++ b/tests/python/sbom/test_gates.py @@ -0,0 +1,283 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""The conditions that must stop a wheel from being published.""" + +from __future__ import annotations + +import json +import shutil +import zipfile + +import pytest +import synth +from sbom import build as build_module +from sbom import licensing +from sbom import validate as validate_module + +DIST_INFO = synth.DIST_INFO +WHEEL_NAME = synth.WHEEL_NAME + + +def _rebuild(workspace, payload): + synth.write_wheel(workspace.wheel, payload) + return build_module.build( + workspace.root, workspace.build, workspace.wheel, workspace.root / "sbom" + ) + + +def test_a_member_from_nowhere_blocks_the_build(workspace): + """A file that no build step and no fetched input explains is a coverage hole.""" + payload = synth.wheel_payload(workspace) + payload["isaaccapture/mystery/libmystery.so.7"] = synth.elf_shared_object( + "libmystery.so.7" + ) + + with pytest.raises(build_module.BuildError, match=r"libmystery\.so\.7"): + _rebuild(workspace, payload) + + +def test_a_component_with_no_license_text_blocks_the_build(workspace): + (workspace.build / "_deps/alpha-src/LICENSE").unlink() + + with pytest.raises(build_module.BuildError, match="alpha"): + _rebuild(workspace, synth.wheel_payload(workspace)) + + +def test_an_empty_license_file_does_not_count_as_evidence(workspace): + # delta reaches the wheel through a static archive, so emptying its license + # leaves the wheel's contents untouched and isolates the evidence gate. + (workspace.build / "_deps/delta-src/BSD-LICENSE").write_text( + " \n", encoding="utf-8" + ) + + with pytest.raises(build_module.BuildError, match="delta"): + _rebuild(workspace, synth.wheel_payload(workspace)) + + +def test_a_library_the_host_cannot_resolve_is_not_claimed_as_the_host_s(workspace): + """Resolving the SONAME is the evidence for "the machine supplied this". + Without it the member is unexplained, and must be reported that way rather + than attributed to a library nobody found.""" + payload = synth.wheel_payload(workspace) + payload["isaaccapture.libs/libunknown-a1b2c3.so"] = synth.elf_shared_object( + "libunknown.so.1", ("libc.so.6",) + ) + synth.write_wheel(workspace.wheel, payload) + + with pytest.raises(build_module.BuildError, match="could not be traced"): + build_module.build( + workspace.root, + workspace.build, + workspace.wheel, + workspace.root / "sbom", + system_resolver=lambda soname: { + "soname": soname, + "resolved_path": None, + "package": None, + "copyright": None, + }, + ) + + +def test_the_build_host_is_the_last_resort_not_a_directory_name(workspace): + """A library the build produced must be attributed to the build even when it + sits in a repair tool's directory: the host lookup runs only once every other + index has failed to explain the bytes.""" + produced = (workspace.build / "lib/libgamma_renamed.so").read_bytes() + payload = synth.wheel_payload(workspace) + payload["isaaccapture.libs/libgamma_renamed-abc123.so"] = produced + staged = workspace.staged / "isaaccapture.libs" + staged.mkdir(parents=True, exist_ok=True) + (staged / "libgamma_renamed-abc123.so").write_bytes(produced) + _rebuild(workspace, payload) + + stem = WHEEL_NAME.removesuffix(".whl") + evidence = json.loads( + (workspace.root / "sbom" / f"{stem}.build-evidence.json").read_text() + ) + entry = next( + item + for item in evidence["attributions"] + if item["path"] == "isaaccapture.libs/libgamma_renamed-abc123.so" + ) + + assert entry["origin"] == "built" + assert entry["primary"] == "gamma" + + +def test_a_vendored_library_is_attributed_to_the_package_that_supplied_it( + workspace, tmp_path +): + copyright_file = tmp_path / "copyright" + copyright_file.write_text(synth.spdx_text("Zlib"), encoding="utf-8") + payload = synth.wheel_payload(workspace) + payload["isaaccapture.libs/libz-a1b2c3.so"] = synth.elf_shared_object( + "libz.so.1", ("libc.so.6",) + ) + synth.write_wheel(workspace.wheel, payload) + + build_module.build( + workspace.root, + workspace.build, + workspace.wheel, + workspace.root / "sbom", + system_resolver=lambda soname: { + "soname": soname, + "resolved_path": "/usr/lib/libz.so.1.3", + "package": "zlib1g", + "copyright": str(copyright_file), + }, + ) + + report = json.loads( + ( + workspace.root / "sbom" / f"{WHEEL_NAME.removesuffix('.whl')}.licenses.json" + ).read_text() + ) + vendored = next( + item for item in report["components"] if item["component"] == "system:libz.so.1" + ) + assert vendored["license_concluded"] == "Zlib" + assert vendored["supplier"] == "Organization: zlib1g" + + +def _replace_member(wheel, name, data): + with zipfile.ZipFile(wheel) as archive: + payload = {item: archive.read(item) for item in archive.namelist()} + payload[name] = data + with zipfile.ZipFile(wheel, "w", zipfile.ZIP_DEFLATED) as archive: + for item in sorted(payload): + archive.writestr(item, payload[item]) + + +def test_editing_a_file_after_the_fact_fails_verification(built): + _replace_member(built["wheel"], "isaaccapture/__init__.py", b"# tampered\n") + + failures = validate_module.check(built["wheel"]) + + assert any("SHA256 mismatch" in failure for failure in failures) + assert any("RECORD digest mismatch" in failure for failure in failures) + + +def test_adding_a_file_after_the_fact_fails_verification(built): + _replace_member(built["wheel"], "isaaccapture/surprise.py", b"# added later\n") + + failures = validate_module.check(built["wheel"]) + + assert any("not covered by the SBOM" in failure for failure in failures) + + +def test_mutating_the_wheel_breaks_its_manifest_binding(built): + _replace_member(built["wheel"], "isaaccapture/__init__.py", b"# tampered\n") + + failures = validate_module.check( + built["wheel"], + build_evidence=built["evidence_path"], + manifest=built["manifest_path"], + ) + + assert any("modified after its digest was bound" in failure for failure in failures) + + +def test_a_second_document_under_sboms_is_rejected(built): + _replace_member(built["wheel"], f"{DIST_INFO}/sboms/extra.spdx.json", b"{}") + + failures = validate_module.check(built["wheel"]) + + assert any("sboms/" in failure for failure in failures) + + +def test_duplicate_wheel_filenames_cannot_be_merged(tmp_path, built): + first = tmp_path / "a.json" + second = tmp_path / "b.json" + for path in (first, second): + path.write_text(json.dumps(built["manifest"]), encoding="utf-8") + + with pytest.raises(build_module.BuildError, match="published twice"): + build_module.merge_manifests([first, second]) + + +def _release_layout(built, tmp_path): + """The shape publish-sbom-evidence assembles from the matrix artifacts.""" + wheels = tmp_path / "release" / "wheels" + evidence = tmp_path / "release" / "evidence" + wheels.mkdir(parents=True) + evidence.mkdir(parents=True) + (wheels / built["wheel"].name).write_bytes(built["wheel"].read_bytes()) + for sidecar in built["out_dir"].glob("*"): + if sidecar.name != "manifest.json": + (evidence / sidecar.name).write_bytes(sidecar.read_bytes()) + merged = tmp_path / "release" / "isaaccapture-sbom-manifest.json" + merged.write_text( + json.dumps(built["manifest"], indent=2, sort_keys=True), encoding="utf-8" + ) + return merged, wheels, evidence + + +def test_a_complete_release_evidence_set_verifies(built, tmp_path): + merged, wheels, evidence = _release_layout(built, tmp_path) + + assert validate_module.check_set(merged, wheels, evidence) == [] + + +def test_a_missing_sidecar_fails_the_release_set(built, tmp_path): + merged, wheels, evidence = _release_layout(built, tmp_path) + next(evidence.glob("*.licenses.md")).unlink() + + failures = validate_module.check_set(merged, wheels, evidence) + + assert any("is missing" in failure for failure in failures) + + +def test_a_missing_wheel_fails_the_release_set(built, tmp_path): + merged, wheels, evidence = _release_layout(built, tmp_path) + (wheels / built["wheel"].name).unlink() + + failures = validate_module.check_set(merged, wheels, evidence) + + assert any("is missing from" in failure for failure in failures) + + +def test_missing_spdx_license_data_stops_the_build(workspace): + """The reference texts come from the build; without them, nothing publishes.""" + shutil.rmtree(workspace.build / "_deps/license-list-data-src") + + with pytest.raises(licensing.CorpusError, match="SPDX license list data not found"): + _rebuild(workspace, synth.wheel_payload(workspace)) + + +def test_a_malformed_reuse_tag_does_not_reach_the_document(workspace): + """A hand-written tag can be unparseable; emitting it verbatim would fail + the published SBOM's own conformance check with no local signal.""" + payload = synth.wheel_payload(workspace) + good = "# SPDX-" + "License-Identifier: mit or Apache-2.0\n" + bad = "# SPDX-" + "License-Identifier: !!! not an expression\n" + for name, text in (("tagged_ok.py", good), ("tagged_bad.py", bad)): + payload[f"isaaccapture/{name}"] = text.encode() + # Authored sources, staged into the package like any other .py. + (workspace.root / "src/python/isaaccapture" / name).write_text(text) + (workspace.staged / "isaaccapture" / name).write_text(text) + _rebuild(workspace, payload) + + with zipfile.ZipFile(workspace.wheel) as archive: + spdx = json.loads(archive.read(f"{DIST_INFO}/sboms/{WHEEL_NAME}.spdx.json")) + concluded = { + item["fileName"].removeprefix("./"): item["licenseConcluded"] + for item in spdx["files"] + } + + # Normalised through the SPDX grammar, not passed through as written. + assert concluded["isaaccapture/tagged_ok.py"] == "MIT OR Apache-2.0" + assert concluded["isaaccapture/tagged_bad.py"] == "NOASSERTION" + + +def test_building_twice_on_the_same_wheel_is_refused(built): + """A second pass would duplicate members rather than replace them.""" + with pytest.raises(build_module.BuildError, match="already carries"): + build_module.build( + built["workspace"].root, + built["workspace"].build, + built["wheel"], + built["out_dir"], + ) diff --git a/tests/python/sbom/test_licensing.py b/tests/python/sbom/test_licensing.py new file mode 100644 index 0000000000..4b0caf452d --- /dev/null +++ b/tests/python/sbom/test_licensing.py @@ -0,0 +1,251 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Licenses are identified from the text a component ships, or not at all.""" + +from __future__ import annotations + +import synth +from sbom import licensing + + +def test_corpus_loads(): + """The whole SPDX list is available, so no subset has to be maintained.""" + version, entries = licensing.corpus() + + assert version.startswith("3.") + assert len(entries) > 400 + assert {"MIT", "Apache-2.0", "BSD-3-Clause", "Zlib", "Qhull"} <= set(entries) + + +def test_deprecated_identifiers_are_not_offered(): + """Naming a component with a withdrawn identifier would be reporting a + license SPDX no longer recognises.""" + _, entries = licensing.corpus() + + assert "GPL-2.0" not in entries, "superseded by GPL-2.0-only/-or-later" + assert "GPL-2.0-only" in entries + + +def test_an_extended_license_is_not_mistaken_for_the_one_it_contains(): + """ECL-2.0 is Apache-2.0 plus a clause, and CC-BY-NC-4.0 is CC-BY-4.0 plus + one. Both clear containment against the shorter licence's own text, so the + file has to be resolved to the licence that explains all of it.""" + apache = [ + license_id + for license_id, _, _ in licensing.identify(synth.spdx_text("Apache-2.0")) + ] + creative = [ + license_id + for license_id, _, _ in licensing.identify(synth.spdx_text("CC-BY-4.0")) + ] + + assert apache == ["Apache-2.0"] + assert creative == ["CC-BY-4.0"] + + +def test_the_extended_license_still_wins_when_it_is_the_real_one(): + """The resolution has to work in both directions, or it is just a bias.""" + matches = [ + license_id + for license_id, _, _ in licensing.identify(synth.spdx_text("ECL-2.0")) + ] + + assert matches == ["ECL-2.0"] + + +def test_identifies_a_canonical_text(): + matches = licensing.identify(synth.spdx_text("MIT")) + + assert [license_id for license_id, _, _ in matches] == ["MIT"] + + +def test_identifies_a_text_wrapped_in_a_copyright_line(): + text = "Copyright (c) 2019 Some Person\n\n" + synth.spdx_text("Zlib") + + assert [license_id for license_id, _, _ in licensing.identify(text)] == ["Zlib"] + + +def test_a_longer_license_subsumes_the_shorter_one_it_contains(): + """A BSD-3 file contains the BSD-2 text; only BSD-3 should be reported.""" + matches = licensing.identify(synth.spdx_text("BSD-3-Clause")) + + assert [license_id for license_id, _, _ in matches] == ["BSD-3-Clause"] + + +def test_a_dual_licensed_file_reports_both(): + text = synth.spdx_text("Apache-2.0") + "\n\n" + synth.spdx_text("CC-BY-4.0") + + identified = {license_id for license_id, _, _ in licensing.identify(text)} + + assert identified == {"Apache-2.0", "CC-BY-4.0"} + + +def _varied_prose(word_count: int) -> str: + """Filler whose wording does not repeat, the way real agreements read.""" + alphabet = "abcdefghijklmnopqrstuvwxyz" + return " ".join( + "term" + + alphabet[index % 26] + + alphabet[(index // 26) % 26] + + alphabet[(index // 676) % 26] + for index in range(word_count) + ) + + +def test_a_license_quoted_in_an_appendix_is_not_the_document_s_license(): + """Large agreements reproduce whole OSS licenses; that is not their license.""" + agreement = ( + "PROPRIETARY AGREEMENT\n\n" + + _varied_prose(6000) + + "\n\nAPPENDIX: third-party notices\n\n" + + synth.spdx_text("BSL-1.0") + ) + + assert licensing.identify(agreement) == [] + + +def test_a_real_license_survives_surrounding_packaging_boilerplate(): + """A Debian copyright file is mostly the license, wrapped in packaging prose.""" + wrapped = ( + "Format: https://www.debian.org/doc/packaging-manuals/copyright-format/1.0/\n" + "Upstream-Name: thing\n\n" + + synth.spdx_text("Zlib") + + "\n\n" + + _varied_prose(200) + ) + + assert [license_id for license_id, _, _ in licensing.identify(wrapped)] == ["Zlib"] + + +def test_unmatched_grant_becomes_a_license_ref_not_a_guess(): + evidence = [ + licensing._evidence( + "demo", "component-file", "demo/LICENSE", synth.PROPRIETARY_LICENSE, "grant" + ) + ] + + concluded, _ = licensing.expression(evidence, "demo") + + assert concluded == "LicenseRef-demo" + + +def test_no_grant_at_all_is_noassertion(): + assert licensing.expression([], "demo") == ("NOASSERTION", "NOASSERTION") + + +def test_finds_a_prefixed_license_file(tmp_path): + (tmp_path / "BSD-LICENSE").write_text( + synth.spdx_text("BSD-3-Clause"), encoding="utf-8" + ) + + evidence = licensing.discover_in_tree("ccd", tmp_path, "ccd-src") + + assert [item.identified for item in evidence] == ["BSD-3-Clause"] + + +def test_a_reuse_pool_does_not_become_the_project_license(tmp_path): + """LICENSES/ holds texts for individual files, not the project's own grant.""" + (tmp_path / "LICENSE").write_text(synth.spdx_text("Apache-2.0"), encoding="utf-8") + pool = tmp_path / "LICENSES" + pool.mkdir() + (pool / "MIT.txt").write_text(synth.spdx_text("MIT"), encoding="utf-8") + (pool / "Zlib.txt").write_text(synth.spdx_text("Zlib"), encoding="utf-8") + + evidence = licensing.discover_in_tree("openxr", tmp_path, "openxr-src") + concluded, _ = licensing.expression(evidence, "openxr") + + assert concluded == "Apache-2.0" + assert sum(1 for item in evidence if item.kind == "pool") == 2, ( + "pool texts still ship" + ) + + +def test_falls_back_to_a_readme_section_when_no_license_file_exists(tmp_path): + (tmp_path / "README.md").write_text( + "# Thing\n\nSome prose.\n\n## Licence\nPublic domain or MIT License.\n", + encoding="utf-8", + ) + + evidence = licensing.discover_in_tree("thing", tmp_path, "thing-src") + + assert [item.origin for item in evidence] == ["readme-section"] + assert licensing.expression(evidence, "thing")[0] == "LicenseRef-thing" + + +def test_notices_are_packaged_but_do_not_set_the_expression(tmp_path): + (tmp_path / "LICENSE").write_text(synth.spdx_text("Apache-2.0"), encoding="utf-8") + (tmp_path / "NOTICE").write_text( + "This product includes software from X.\n", encoding="utf-8" + ) + + evidence = licensing.discover_in_tree("thing", tmp_path, "thing-src") + + assert {item.kind for item in evidence} == {"grant", "notice"} + assert licensing.expression(evidence, "thing")[0] == "Apache-2.0" + + +def test_an_upstream_spdx_tag_is_reported_as_declared(tmp_path): + # REUSE-IgnoreStart + (tmp_path / "LICENSE").write_text( + "SPDX-License-Identifier: MIT\n\n" + synth.spdx_text("MIT"), encoding="utf-8" + ) + # REUSE-IgnoreEnd + + evidence = licensing.discover_in_tree("thing", tmp_path, "thing-src") + concluded, declared = licensing.expression(evidence, "thing") + + assert (concluded, declared) == ("MIT", "MIT") + + +def test_a_component_carrying_two_licenses_combines_them(): + """Two identified grants must AND together, not collapse to NOASSERTION. + + The combiner returns a parsed expression object, and asking such an object + for its truth value raises rather than answering -- so this path has to be + exercised with more than one licence. + """ + evidence = [ + licensing._evidence( + "dual", + "component-file", + "dual/LICENSE", + synth.spdx_text("Apache-2.0") + "\n\n" + synth.spdx_text("CC-BY-4.0"), + "grant", + ) + ] + + concluded, _ = licensing.expression(evidence, "dual") + + assert concluded == "Apache-2.0 AND CC-BY-4.0" + + +def test_an_index_of_licenses_is_not_a_grant(tmp_path): + """A cone-mode sparse checkout keeps root files, so a repository that + merely catalogues licences can present names that look like grants.""" + (tmp_path / "licenses.md").write_text("| MIT | Apache-2.0 |\n", encoding="utf-8") + (tmp_path / "accessingLicenses.md").write_text( + "how to fetch them\n", encoding="utf-8" + ) + + assert licensing.discover_in_tree("catalogue", tmp_path, "catalogue-src") == [] + + +def test_a_project_title_is_not_a_license_section(tmp_path): + """`# License List Data` heads a README; it does not grant anything.""" + (tmp_path / "README.md").write_text( + "# License List Data\n\nGenerated data.\n", encoding="utf-8" + ) + + assert licensing.discover_in_tree("catalogue", tmp_path, "catalogue-src") == [] + + +def test_a_real_license_section_is_still_found(tmp_path): + (tmp_path / "README.md").write_text( + "# Thing\n\nprose\n\n## Licence\nPublic domain or MIT License.\n", + encoding="utf-8", + ) + + evidence = licensing.discover_in_tree("thing", tmp_path, "thing-src") + + assert [item.origin for item in evidence] == ["readme-section"] diff --git a/tests/python/sbom/test_spdx_conformance.py b/tests/python/sbom/test_spdx_conformance.py new file mode 100644 index 0000000000..15f38d836b --- /dev/null +++ b/tests/python/sbom/test_spdx_conformance.py @@ -0,0 +1,34 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Third-party conformance check, so the document is valid SPDX and not just ours.""" + +from __future__ import annotations + +import pytest + +pytest.importorskip("spdx_tools", reason="spdx-tools is a dev dependency of this suite") + +from spdx_tools.spdx.parser.parse_anything import parse_file # noqa: E402 +from spdx_tools.spdx.validation.document_validator import ( # noqa: E402 + validate_full_spdx_document, +) + + +def test_emitted_document_is_valid_spdx_2_3(built): + document = parse_file(str(built["spdx_path"])) + + messages = validate_full_spdx_document(document) + + assert [message.validation_message for message in messages] == [] + + +def test_relationships_survive_the_round_trip(built): + document = parse_file(str(built["spdx_path"])) + + kinds = { + relationship.relationship_type.name for relationship in document.relationships + } + + assert {"DESCRIBES", "STATIC_LINK", "CONTAINS", "DEPENDS_ON"} <= kinds + assert {"GENERATED_FROM", "COPY_OF"} & kinds diff --git a/tests/python/sbom/test_wheel_sbom.py b/tests/python/sbom/test_wheel_sbom.py new file mode 100644 index 0000000000..a9a2734ccb --- /dev/null +++ b/tests/python/sbom/test_wheel_sbom.py @@ -0,0 +1,251 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""End to end: a repaired wheel goes in, a publishable wheel comes out.""" + +from __future__ import annotations + +import json +import zipfile + +import synth +from sbom import build as build_module +from sbom import validate as validate_module +from sbom import wheelfile + +DIST_INFO = synth.DIST_INFO +WHEEL_NAME = synth.WHEEL_NAME + + +def _names(wheel): + with zipfile.ZipFile(wheel) as archive: + return set(archive.namelist()) + + +def _embedded(wheel): + with zipfile.ZipFile(wheel) as archive: + return json.loads(archive.read(f"{DIST_INFO}/sboms/{WHEEL_NAME}.spdx.json")) + + +def test_wheel_verifies_against_its_own_sbom(built): + assert validate_module.check(built["wheel"]) == [] + + +def test_release_validation_passes(built): + failures = validate_module.check( + built["wheel"], + build_evidence=built["evidence_path"], + manifest=built["manifest_path"], + ) + assert failures == [] + + +def test_sbom_lands_in_dist_info_sboms_and_nothing_else_does(built): + sboms = {name for name in _names(built["wheel"]) if f"{DIST_INFO}/sboms/" in name} + + assert sboms == {f"{DIST_INFO}/sboms/{WHEEL_NAME}.spdx.json"} + + +def test_every_wheel_file_is_covered(built): + spdx = _embedded(built["wheel"]) + documented = {item["fileName"].removeprefix("./") for item in spdx["files"]} + excluded = { + item.removeprefix("./") + for package in spdx["packages"] + for item in package.get("packageVerificationCode", {}).get( + "packageVerificationCodeExcludedFiles", [] + ) + } + + assert _names(built["wheel"]) - documented == excluded + + +def test_every_member_records_where_it_came_from(built): + evidence = json.loads(built["evidence_path"].read_text(encoding="utf-8")) + origins = {item["path"]: item["origin"] for item in evidence["attributions"]} + + assert origins["isaaccapture/__init__.py"] == "repo-source" + assert origins["isaaccapture/_generated.py"] == "generated" + assert origins["isaaccapture/_ext.cpython-312-x86_64-linux-gnu.so"] == "built" + assert origins["isaaccapture/viz/libgamma_renamed.so"] == "built" + assert origins["isaaccapture/viz/GAMMA_LICENSE"] == "copied" + assert origins["isaaccapture/sdk/native/libdemo.so"] == "archive-copy" + assert origins["vendorpy/mod.py"] == "copied" + + +def test_post_processed_files_are_recorded_as_derived_not_as_copies(built): + """patchelf and the MJCF rewrite change bytes; the claim has to change too.""" + evidence = json.loads(built["evidence_path"].read_text(encoding="utf-8")) + origins = {item["path"]: item for item in evidence["attributions"]} + + patched = origins["isaaccapture/sdk/native/libextra.so"] + assert patched["origin"] == "archive-derived" + assert "modified copy of" in patched["detail"] + + rewritten = origins["vendorpy/assets/hand.xml"] + assert rewritten["origin"] == "derived" + + +def test_components_are_discovered_not_declared(built): + spdx = _embedded(built["wheel"]) + names = {package["name"] for package in spdx["packages"]} + + assert {"alpha", "beta", "gamma", "delta", "vendorpy"} <= names + assert "demosdk-1.2.0-linux-amd64" in names + assert "omega" not in names, "a test-only dependency is not in the wheel" + + +def test_a_dependency_reaching_no_wheel_file_is_recorded_as_excluded(built): + report = json.loads(built["report_path"].read_text(encoding="utf-8")) + excluded = [item["component"] for item in report["exclusions"]] + + assert "omega" in excluded, "a test-only dependency ships nothing" + # The SPDX reference data is fetched like any other dependency, so it is + # inventoried like any other dependency -- and excluded for the same reason. + assert "license-list-data" in excluded + + +def test_licenses_are_packaged_and_declared(built): + names = _names(built["wheel"]) + third_party = { + name for name in names if f"{DIST_INFO}/licenses/third-party/" in name + } + + assert f"{DIST_INFO}/licenses/THIRD-PARTY-NOTICES.md" in names + assert any("/alpha/" in name for name in third_party) + assert any("demosdk" in name for name in third_party) + + raw, metadata = wheelfile.read_metadata(built["wheel"], DIST_INFO) + declared = set(metadata.get("license_files") or []) + # PEP 639 states these relative to .dist-info/licenses/, so each must + # resolve back to a member that is actually in the wheel. + prefix = f"{DIST_INFO}/licenses/" + assert {name.removeprefix(prefix) for name in third_party} <= declared + for value in declared: + assert f"{prefix}{value}" in names, ( + f"License-File {value!r} resolves to nothing" + ) + assert "LICENSE.md" in declared, ( + "the project's own license must survive the rewrite" + ) + + # No RFC 822 folding: a wrapped path is valid, and packaging unfolds it, but + # nothing else writes one and a continuation line defeats simpler readers. + assert not [ + line for line in raw.decode().splitlines() if line.startswith((" ", "\t")) + ] + + +def test_an_unidentified_grant_travels_verbatim_in_the_document(built): + spdx = _embedded(built["wheel"]) + extracted = {item["licenseId"]: item for item in spdx["hasExtractedLicensingInfos"]} + + assert "LicenseRef-demosdk-1.2.0-linux-amd64" in extracted + assert ( + "DEMO CORP" + in extracted["LicenseRef-demosdk-1.2.0-linux-amd64"]["extractedText"] + ) + + +def test_reuse_tags_on_shipped_files_are_reported_per_file(built): + spdx = _embedded(built["wheel"]) + tagged = { + item["fileName"].removeprefix("./"): item.get("licenseConcluded") + for item in spdx["files"] + } + + assert tagged["vendorpy/mod.py"] == "Apache-2.0" + assert tagged["isaaccapture/__init__.py"] == "Apache-2.0" + + +def test_record_is_consistent_after_the_rewrite(built): + record = wheelfile.read_record(built["wheel"], DIST_INFO) + with zipfile.ZipFile(built["wheel"]) as archive: + members = set(archive.namelist()) + for name, (digest, size) in record.items(): + if name == f"{DIST_INFO}/RECORD": + continue + data = archive.read(name) + assert digest == wheelfile.record_hash(data), name + assert size == str(len(data)), name + assert set(record) == members + + +def test_manifest_binds_the_final_wheel_digest(built): + entry = built["manifest"]["wheels"][0] + + assert entry["filename"] == WHEEL_NAME + assert entry["sha256"] == build_module.sha256_file(built["wheel"]) + assert entry["sbom_in_wheel"] == f"{DIST_INFO}/sboms/{WHEEL_NAME}.spdx.json" + + +def test_published_copy_matches_the_embedded_one(built): + with zipfile.ZipFile(built["wheel"]) as archive: + embedded = archive.read(f"{DIST_INFO}/sboms/{WHEEL_NAME}.spdx.json") + + assert built["spdx_path"].read_bytes() == embedded + + +def test_external_runtime_dependencies_are_not_claimed_as_contents(built): + spdx = _embedded(built["wheel"]) + external = { + package["name"] + for package in spdx["packages"] + if package["SPDXID"].startswith("SPDXRef-Package-external-") + } + + assert {"libc.so.6", "libstdc++.so.6"} <= external + assert "libdemo.so" not in external, "a library inside the wheel is not external" + + +def test_consumer_requirements_come_from_wheel_metadata(built): + spdx = _embedded(built["wheel"]) + comments = " ".join( + package.get("comment", "") + for package in spdx["packages"] + if package["SPDXID"].startswith("SPDXRef-Package-pypi-") + ) + + assert "numpy>=1.23.0" in comments + assert "pyyaml>=6.0.3" in comments + + +def test_build_is_reproducible_under_source_date_epoch(tmp_path, monkeypatch): + monkeypatch.setenv("SOURCE_DATE_EPOCH", "1700000000") + + digests = [] + for run in ("a", "b"): + space = synth.create(tmp_path / run) + synth.write_wheel(space.wheel, synth.wheel_payload(space)) + build_module.build(space.root, space.build, space.wheel, space.root / "sbom") + digests.append(build_module.sha256_file(space.wheel)) + + assert digests[0] == digests[1] + + +def test_source_date_epoch_normalizes_timestamps_the_input_wheel_carried( + workspace, monkeypatch +): + """A zip records a local timestamp per member, so a wheel carries its + builder's clock. Under SOURCE_DATE_EPOCH the rewrite has to normalize the + members it copies through as well, or it is reproducible only for an input + wheel that already was.""" + import zipfile as zf + + stamped = synth.wheel_payload(workspace) + with zf.ZipFile(workspace.wheel, "w", zf.ZIP_DEFLATED) as archive: + for name in sorted(stamped): + archive.writestr( + zf.ZipInfo(name, date_time=(2021, 6, 5, 4, 3, 2)), stamped[name] + ) + archive.writestr(zf.ZipInfo("x", date_time=(2021, 6, 5, 4, 3, 2)), b"") + synth.write_wheel(workspace.wheel, stamped) + + monkeypatch.setenv("SOURCE_DATE_EPOCH", "1700000000") + build_module.build( + workspace.root, workspace.build, workspace.wheel, workspace.root / "sbom" + ) + + with zf.ZipFile(workspace.wheel) as archive: + stamps = {info.date_time for info in archive.infolist()} + assert len(stamps) == 1, f"members carry differing timestamps: {stamps}" From 6c73e1944d055fc6ae4afd867a30a9ff7555675c Mon Sep 17 00:00:00 2001 From: Andrei Aristarkhov Date: Wed, 30 Sep 2026 19:49:31 -0700 Subject: [PATCH 005/101] build(sbom): describe the wheel a real build produces Running the collector against a full build rather than the synthetic fixtures turned up four routes it could not explain and two claims it got wrong. auditwheel writes its own PEP 770 document to `.dist-info/sboms/`, so the re-run guard and the verifier now look for this collector's document by name instead of treating the directory as its own. A repair tool rewrites SONAME and RPATH, so a vendored library no longer hashes to the bytes it came from. The GNU build-id does not change, and ties a patched member back to an archive member, a build output, or the build machine's copy. `_deps` holds both fetched sources and this build's own output. Only `*-src` is the component-source domain; the rest is indexed as generated, and a tree a dependency stages there takes the component its byte-identical siblings name. RECORD states size 0 for an empty member, which `or ""` erased -- every wheel carries an empty `py.typed`, so every wheel failed its own check. And zero bytes are identical everywhere, so an empty file is attributed by path rather than by a content match that proves nothing. A licence id newer than the grammar `license-expression` ships cannot be validated by a consumer, so the text travels as a LicenseRef instead. Signed-off-by: Andrei Aristarkhov --- scripts/sbom/sbom/build.py | 9 +- scripts/sbom/sbom/discovery.py | 93 +++++++++++++++++- scripts/sbom/sbom/elf.py | 19 +++- scripts/sbom/sbom/evidence.py | 31 +++++- scripts/sbom/sbom/inventory.py | 136 ++++++++++++++++++++++++++- scripts/sbom/sbom/licensing.py | 22 +++++ scripts/sbom/sbom/validate.py | 14 +-- scripts/sbom/sbom/wheelfile.py | 24 ++++- src/core/AGENTS.md | 2 + tests/python/sbom/CMakeLists.txt | 2 +- tests/python/sbom/conftest.py | 4 +- tests/python/sbom/synth.py | 20 ++++ tests/python/sbom/test_gates.py | 34 ++++++- tests/python/sbom/test_wheel_sbom.py | 8 +- 14 files changed, 388 insertions(+), 30 deletions(-) diff --git a/scripts/sbom/sbom/build.py b/scripts/sbom/sbom/build.py index ae73dbc681..e5f7d6cf86 100644 --- a/scripts/sbom/sbom/build.py +++ b/scripts/sbom/sbom/build.py @@ -80,12 +80,10 @@ def build( licensing.load_corpus(license_data or default_license_data(build_dir)) discovery = evidence_module.discover(repo_root, build_dir) wheel = wheelfile.scan(wheel_path, analyze_elf=True) - already = [ - item.name for item in wheel.entries if f"{wheel.dist_info}/sboms/" in item.name - ] - if already: + sbom_name = wheelfile.sbom_member(wheel.dist_info, wheel_path.name) + if any(item.name == sbom_name for item in wheel.entries): raise BuildError( - f"{wheel_path.name} already carries {already[0]}. Run this on a freshly " + f"{wheel_path.name} already carries {sbom_name}. Run this on a freshly " "repaired wheel; rewriting one twice would duplicate its members." ) metadata_raw, _ = wheelfile.read_metadata(wheel_path, wheel.dist_info) @@ -140,7 +138,6 @@ def build( metadata_declared = patched_metadata != metadata_raw replacements = {f"{wheel.dist_info}/METADATA": patched_metadata} - sbom_name = f"{wheel.dist_info}/sboms/{wheel_path.name}.spdx.json" excluded = [f"{wheel.dist_info}/RECORD", sbom_name] projected = [item for item in wheel.entries if item.name not in replacements] + [ diff --git a/scripts/sbom/sbom/discovery.py b/scripts/sbom/sbom/discovery.py index 8a8ec62afe..e45291a14b 100644 --- a/scripts/sbom/sbom/discovery.py +++ b/scripts/sbom/sbom/discovery.py @@ -17,6 +17,7 @@ from __future__ import annotations +import functools import glob import hashlib import json @@ -31,7 +32,7 @@ from packageurl import PackageURL -from . import licensing +from . import elf, licensing # Directories that never hold an input worth indexing. _SKIP_DIRS = { @@ -556,6 +557,82 @@ def discover_vendored( ) +def _host_library_paths() -> list[Path]: + """Every shared library the loader can see, plus the CUDA trees it cannot.""" + paths: set[Path] = set() + if shutil.which("ldconfig"): + try: + result = subprocess.run( + ["ldconfig", "-p"], + capture_output=True, + text=True, + check=False, + timeout=60, + ) + for line in result.stdout.splitlines(): + _, _, target = line.partition("=>") + target = target.strip() + if target: + paths.add(Path(target)) + except (OSError, subprocess.SubprocessError): + pass + roots = [Path(item) for item in _SEARCH_ROOTS if Path(item).is_dir()] + roots += [Path(item) for item in sorted(glob.glob("/usr/local/cuda*/lib64"))] + for root in roots: + paths.update(item for item in root.glob("*.so*") if item.is_file()) + return sorted(paths) + + +@functools.lru_cache(maxsize=1) +def _host_build_id_index() -> dict[str, str]: + """Host libraries by GNU build-id. + + A repair tool rewrites SONAME on the way into the wheel, so the name is no + longer a way back to the machine's copy. The build-id is, and it is the same + note the linker wrote. Built once; reading it from every library the loader + knows costs about a second. + """ + index: dict[str, str] = {} + for path in _host_library_paths(): + try: + data = path.read_bytes() + except OSError: + continue + if not elf.is_elf(data[:4]): + continue + try: + build_id = elf.read_dynamic(data).build_id + except Exception: # noqa: BLE001 - a malformed host library is not ours to fix + continue + if not build_id: + continue + # A regular file beats a symlink to it: dpkg-query knows the real path. + current = index.get(build_id) + if current is None or ( + path.is_symlink() is False and Path(current).is_symlink() + ): + index[build_id] = str(path) + return index + + +def resolve_system_library_by_build_id(build_id: str) -> dict: + """Trace a vendored library back by build-id, whatever it was renamed to.""" + origin: dict = { + "soname": None, + "build_id": build_id, + "resolved_path": None, + "package": None, + "copyright": None, + } + found = _host_build_id_index().get(build_id) + if found is None: + return origin + library = Path(found).resolve() + origin["soname"] = library.name + origin["resolved_path"] = str(library) + return _identify_host_package(library, origin) + + def resolve_system_library(soname: str) -> dict: """Trace an auditwheel-vendored library back to the package that supplied it.""" origin: dict = { @@ -572,7 +649,11 @@ def resolve_system_library(soname: str) -> dict: return origin library = library.resolve() origin["resolved_path"] = str(library) + return _identify_host_package(library, origin) + +def _identify_host_package(library: Path, origin: dict) -> dict: + """Name the distro package that owns a resolved host library.""" if shutil.which("dpkg-query") is None: return origin try: @@ -674,6 +755,9 @@ class ArchiveIndex: def __init__(self) -> None: self.by_hash: dict[str, Member] = {} self.by_name: dict[str, list[Member]] = {} + # Survives the RPATH/SONAME rewrite a repair tool applies on the way in, + # so a patched wheel member still points at the archive it came from. + self.by_build_id: dict[str, Member] = {} self.archives: dict[str, dict] = {} def add_archive(self, path: Path, display: str) -> None: @@ -691,6 +775,13 @@ def add_archive(self, path: Path, display: str) -> None: for member, data in members: self.by_hash.setdefault(member.sha256, member) self.by_name.setdefault(Path(member.path).name, []).append(member) + if elf.is_elf(data[:4]): + try: + build_id = elf.read_dynamic(data).build_id + except Exception: # noqa: BLE001 - a malformed member is not ours to fix + build_id = None + if build_id: + self.by_build_id.setdefault(build_id, member) self._maybe_license(display, member, data) def _maybe_license(self, display: str, member: Member, data: bytes) -> None: diff --git a/scripts/sbom/sbom/elf.py b/scripts/sbom/sbom/elf.py index 0a0f7b69f6..4a23343e7e 100644 --- a/scripts/sbom/sbom/elf.py +++ b/scripts/sbom/sbom/elf.py @@ -1,7 +1,7 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -"""SONAME and NEEDED for the shared libraries a wheel carries. +"""SONAME, NEEDED and the GNU build-id for the shared libraries a wheel carries. Parsing is pyelftools' job, not ours -- it is the same library auditwheel uses to repair these wheels, so the tool that wrote the ELF and the tool that reads @@ -30,6 +30,10 @@ class DynamicInfo: soname: str | None needed: tuple[str, ...] machine: str + # Identifies the bytes the linker produced, and patchelf leaves it alone. + # So it still matches after a repair tool rewrites SONAME and RPATH, which + # is what makes it the only link back to a library's pristine origin. + build_id: str | None = None class NotAnElf(Exception): @@ -41,7 +45,7 @@ def is_elf(data: bytes) -> bool: def read_dynamic(data: bytes) -> DynamicInfo: - """Parse SONAME and NEEDED out of an in-memory ELF image.""" + """Parse SONAME, NEEDED and the build-id out of an in-memory ELF image.""" if not is_elf(data): raise NotAnElf("missing ELF magic") @@ -61,8 +65,19 @@ def read_dynamic(data: bytes) -> DynamicInfo: elif kind == "DT_NEEDED": needed.append(tag.needed) + build_id: str | None = None + # Segment, not section, for the same reason as above. + for segment in elffile.iter_segments(type="PT_NOTE"): + for note in segment.iter_notes(): + if note["n_type"] == "NT_GNU_BUILD_ID": + build_id = note["n_desc"] + break + if build_id: + break + return DynamicInfo( soname=soname, needed=tuple(needed), machine=_ARCH.get(machine, machine), + build_id=build_id, ) diff --git a/scripts/sbom/sbom/evidence.py b/scripts/sbom/sbom/evidence.py index 0a85414f0d..1ae87dffc3 100644 --- a/scripts/sbom/sbom/evidence.py +++ b/scripts/sbom/sbom/evidence.py @@ -26,6 +26,7 @@ discover_vendored, project_authors, discover_source_trees, + _walk, ) # Cache entries that pin what a wheel contains. The rest of the cache is paths on @@ -56,6 +57,8 @@ class Discovery: repo_files: FileIndex build_files: FileIndex ownership: Ownership + # build_files display prefix -> component whose tree the build staged there. + staged_component_dirs: dict[str, str] @property def staged_root(self) -> Path | None: @@ -161,11 +164,36 @@ def discover(repo_root: Path, build_dir: Path) -> Discovery: skip=(build_dir, repo_root / "build-wheel", repo_root / "dist"), ) + # _deps/*-src is the component-source domain, already indexed above; a wheel + # member matching one is a copy of upstream, not something this build made. + # Everything else under _deps is this build's own output -- FetchContent + # build trees, and trees a dependency stages for packaging -- so it belongs + # in build_files like any other generated file. The staged tree stays out: + # indexing it would let the wheel explain itself. build_files = FileIndex() build_files.add_tree( - build_dir, "build", skip=(deps_dir, build_dir / "python_package") + build_dir, + "build", + skip=(*sorted(deps_dir.glob("*-src")), build_dir / "python_package"), ) + # A dependency may stage a packaged copy of its own tree under _deps//. + # Files the build transformed on the way in no longer hash to the component, + # so infer the directory's owner from the siblings that still do; one clear + # owner or nothing, since a guess here would invent a license obligation. + staged_component_dirs: dict[str, str] = {} + for root in sorted(deps_dir.iterdir()): + if not root.is_dir() or root.name.endswith("-src"): + continue + owners = set() + for path in _walk(root): + match = source_files.by_hash.get(build_files.digest_of(path) or "") + if match: + owners.add(match.split("/", 1)[0].removesuffix("-src")) + if len(owners) == 1: + display = f"build/{root.relative_to(build_dir).as_posix()}" + staged_component_dirs[display] = owners.pop() + # Third-party code checked in here rather than fetched: the candidates are # what the build actually compiles or includes, so nothing outside the build # is inspected. @@ -209,6 +237,7 @@ def discover(repo_root: Path, build_dir: Path) -> Discovery: repo_files=repo_files, build_files=build_files, ownership=ownership, + staged_component_dirs=staged_component_dirs, ) diff --git a/scripts/sbom/sbom/inventory.py b/scripts/sbom/sbom/inventory.py index b324a1b56f..fe9c96ebfc 100644 --- a/scripts/sbom/sbom/inventory.py +++ b/scripts/sbom/sbom/inventory.py @@ -15,10 +15,13 @@ from pathlib import Path from typing import TYPE_CHECKING -from . import licensing +import hashlib + +from . import elf, licensing from .discovery import ( Component, resolve_system_library, + resolve_system_library_by_build_id, sha256_file, system_component, ) @@ -85,6 +88,9 @@ def files_of(self, key: str) -> list[str]: ) +_EMPTY_SHA256 = hashlib.sha256(b"").hexdigest() + + class Resolver: """Probes a candidate's bytes and path against every discovered index.""" @@ -100,7 +106,9 @@ def __init__(self, discovery: Discovery, system_resolver=None) -> None: self.build_files = discovery.build_files self.ownership = discovery.ownership self.source_files = discovery.source_files + self.staged_component_dirs = discovery.staged_component_dirs self._artifacts_by_hash: dict[str, list] = {} + self._artifacts_by_build_id: dict[str, list] = {} for artifact in self.graph.artifacts.values(): if artifact.output.is_file(): # build_files already hashed everything under the build tree. @@ -108,6 +116,11 @@ def __init__(self, discovery: Discovery, system_resolver=None) -> None: artifact.output ) self._artifacts_by_hash.setdefault(digest, []).append(artifact) + build_id = _build_id_of(artifact.output) + if build_id: + self._artifacts_by_build_id.setdefault(build_id, []).append( + artifact + ) def _from_artifact(self, artifact, path: str, detail: str) -> Attribution: contributions = self.graph.contributions(artifact, self.ownership) @@ -136,6 +149,7 @@ def resolve( digest: str, probe: Path | None, soname: str | None = None, + build_id: str | None = None, ) -> Attribution | None: """Explain one member: by content, then by path, then by the machine.""" candidate_digest = digest @@ -144,6 +158,13 @@ def resolve( candidate_digest = sha256_file(probe) source = f"staged at {probe.relative_to(self.build_dir).as_posix()}" + # Zero bytes are identical everywhere, so a content match proves nothing: + # it would hand an empty marker file to whichever component happened to be + # indexed first, and invent that component's license obligation with it. + # The path is the only evidence such a file carries. + if candidate_digest == _EMPTY_SHA256: + return self._resolve_by_path(wheel_path, source) + artifacts = self._artifacts_by_hash.get(candidate_digest) if artifacts: return self._from_artifact( @@ -179,14 +200,40 @@ def resolve( build_file = self.build_files.by_hash.get(candidate_digest) if build_file: + # An empty file hashes the same as every other empty file, so the + # first indexed path wins and says nothing. Where the build tree + # holds the same bytes under the wheel's own path, that is the + # better answer. + if Path(build_file).name != Path(wheel_path).name: + by_path = self.build_files.path_suffix_match(wheel_path) + if by_path: + build_file = by_path + owner = self._staged_component_of(build_file) return Attribution( - path=wheel_path, origin="generated", detail=f"{source}; {build_file}" + path=wheel_path, + origin="generated", + detail=f"{source}; {build_file}", + components={owner: {"derived-file"}} if owner else {}, + primary=owner, ) + # Content no longer matches, so the bytes were patched on the way in. + # The build-id is what the linker wrote and patchelf leaves alone, which + # makes it evidence where a matching file name is only a coincidence. + if build_id: + by_build_id = self._resolve_by_build_id(wheel_path, source, build_id) + if by_build_id is not None: + return by_build_id + by_path = self._resolve_by_path(wheel_path, source) if by_path is not None: return by_path + if build_id: + from_host = self._from_build_host_by_build_id(wheel_path, build_id) + if from_host is not None: + return from_host + # Nothing this build produced or fetched explains these bytes. A shared # library reaching that point came from the machine -- whatever the # repair tool chose to call its directory. @@ -194,6 +241,13 @@ def resolve( return self._from_build_host(wheel_path, soname) return None + def _staged_component_of(self, build_file: str) -> str | None: + """The component whose tree this build staged under _deps//.""" + for prefix, key in self.staged_component_dirs.items(): + if build_file == prefix or build_file.startswith(prefix + "/"): + return key + return None + def _from_build_host(self, wheel_path: str, soname: str) -> Attribution | None: """Claim host origin only if the host actually has this library. @@ -221,6 +275,55 @@ def _from_build_host(self, wheel_path: str, soname: str) -> Attribution | None: primary=component.key, ) + def _resolve_by_build_id( + self, wheel_path: str, source: str, build_id: str + ) -> Attribution | None: + """The pristine bytes this member was patched from, named by build-id.""" + member = self.archives.by_build_id.get(build_id) + if member: + return Attribution( + path=wheel_path, + origin="archive-derived", + detail=( + f"{source}; build-id {build_id} matches {member.path} in " + f"{member.container} (origin sha256:{member.sha256})" + ), + components={member.container: {"extracted-file"}}, + primary=member.container, + ) + + artifacts = self._artifacts_by_build_id.get(build_id) + if artifacts: + return self._from_artifact( + artifacts[0], wheel_path, f"{source}; build-id {build_id}, patched" + ) + return None + + def _from_build_host_by_build_id( + self, wheel_path: str, build_id: str + ) -> Attribution | None: + """The machine's own copy, found by build-id after a SONAME rewrite.""" + origin = resolve_system_library_by_build_id(build_id) + if not origin.get("resolved_path"): + return None + soname = origin["soname"] + self.system_libraries[soname] = origin + component = system_component(soname, origin) + self.components[component.key] = component + + detail = ( + f"build-id {build_id} matches the build machine's {origin['resolved_path']}" + ) + if origin.get("package"): + detail += f", supplied by {origin['package']}" + return Attribution( + path=wheel_path, + origin="build-host-library", + detail=detail, + components={component.key: {"vendored-library"}}, + primary=component.key, + ) + def _resolve_by_path(self, wheel_path: str, source: str) -> Attribution | None: """Post-processing changes bytes; the path still says where they came from. @@ -270,6 +373,17 @@ def _resolve_by_path(self, wheel_path: str, source: str) -> Attribution | None: detail=f"{source}; derived from {repo_file}", ) + build_file = self.build_files.path_suffix_match(wheel_path) + if build_file: + owner = self._staged_component_of(build_file) + return Attribution( + path=wheel_path, + origin="generated", + detail=f"{source}; {build_file}", + components={owner: {"derived-file"}} if owner else {}, + primary=owner, + ) + return None @@ -328,6 +442,20 @@ def build( ) +def _build_id_of(path: Path) -> str | None: + """The build-id of a file on disk, or None if it is not an ELF.""" + try: + data = path.read_bytes() + except OSError: + return None + if not elf.is_elf(data[:4]): + return None + try: + return elf.read_dynamic(data).build_id + except Exception: # noqa: BLE001 - a malformed build output is not ours to fix + return None + + def _attribute( entry: Entry, wheel: WheelInfo, @@ -342,7 +470,9 @@ def _attribute( ) probe = staged_root / entry.name if staged_root is not None else None - return resolver.resolve(entry.name, entry.sha256, probe, entry.soname) + return resolver.resolve( + entry.name, entry.sha256, probe, entry.soname, entry.build_id + ) def _spdx_tag(entry: Entry) -> str | None: diff --git a/scripts/sbom/sbom/licensing.py b/scripts/sbom/sbom/licensing.py index 36d38f3538..3da74e664a 100644 --- a/scripts/sbom/sbom/licensing.py +++ b/scripts/sbom/sbom/licensing.py @@ -392,6 +392,22 @@ def license_ref(component_key: str) -> str: return f"LicenseRef-{spdx_safe(component_key)}" +def expressible(license_id: str) -> bool: + """Whether the SPDX grammar we ship can validate this identifier. + + The corpus the build fetches moves ahead of what `license-expression` + knows -- SPDX 3.29.0 added BSD-2-Clause-pos-unchanged, which no released + version of the grammar recognises. Matching against the newer corpus is + right; naming an ID a consumer's validator will reject is not. + """ + if license_id.startswith("LicenseRef-"): + return True + try: + return not _spdx_licensing().validate(license_id).errors + except (ExpressionError, ValueError, TypeError): + return False + + def expression( evidence: list[LicenseEvidence], component_key: str | None = None ) -> tuple[str, str]: @@ -401,11 +417,17 @@ def expression( grant that matches nothing is not dropped and not guessed at: it becomes a LicenseRef whose text travels in the document, which says "these are the terms we shipped" without claiming to know which license they are. + + An ID the shipped grammar cannot express is treated the same way, and the + whole expression falls back rather than just the offending term: dropping + one ID from `A AND B` would understate the terms. """ grants = [item for item in evidence if item.kind == "grant"] identified = sorted( {license_id for item in grants for license_id, _, _ in item.matches} ) + if identified and component_key and not all(map(expressible, identified)): + identified = [license_ref(component_key)] if not identified and grants and component_key: identified = [license_ref(component_key)] concluded = _combine(identified) if identified else "NOASSERTION" diff --git a/scripts/sbom/sbom/validate.py b/scripts/sbom/sbom/validate.py index cdce3b98d7..05b5673e46 100644 --- a/scripts/sbom/sbom/validate.py +++ b/scripts/sbom/sbom/validate.py @@ -34,13 +34,15 @@ def _embedded_sbom( archive: zipfile.ZipFile, dist_info: str, wheel_name: str ) -> tuple[str, dict]: - name = f"{dist_info}/sboms/{wheel_name}.spdx.json" - names = [ - item for item in archive.namelist() if item.startswith(f"{dist_info}/sboms/") - ] - if names != [name]: + name = wheelfile.sbom_member(dist_info, wheel_name) + if name not in archive.namelist(): + found = [ + item + for item in archive.namelist() + if item.startswith(f"{dist_info}/sboms/") + ] raise LookupError( - f"expected exactly {name} under {dist_info}/sboms/, found {names or 'nothing'}" + f"expected {name}, found {found or 'nothing'} under {dist_info}/sboms/" ) return name, json.loads(archive.read(name)) diff --git a/scripts/sbom/sbom/wheelfile.py b/scripts/sbom/sbom/wheelfile.py index 50be7bc1c4..dba048309d 100644 --- a/scripts/sbom/sbom/wheelfile.py +++ b/scripts/sbom/sbom/wheelfile.py @@ -72,6 +72,7 @@ class Entry: soname: str | None = None needed: tuple[str, ...] = () machine: str | None = None + build_id: str | None = None # Enough to find a REUSE tag, kept so nothing re-opens the archive for it. head: bytes = b"" @@ -89,6 +90,16 @@ class WheelInfo: entries: tuple[Entry, ...] +def sbom_member(dist_info: str, wheel_name: str) -> str: + """Where this collector's SPDX document sits inside the wheel. + + PEP 770 makes `.dist-info/sboms/` a shared directory: auditwheel writes + `auditwheel.cdx.json` there for the libraries it vendors. Match this exact + name, never the directory, or a second tool's document reads as ours. + """ + return f"{dist_info}/sboms/{wheel_name}.spdx.json" + + def record_hash(data: bytes) -> str: """RECORD's hash field, as PEP 376 spells it.""" return str(Hash.parse(f"sha256={_urlsafe_b64(data)}")) @@ -139,13 +150,14 @@ def scan(wheel_path: Path, analyze_elf: bool = False) -> WheelInfo: if info.is_dir(): continue data = archive.read(info.filename) - soname = needed = machine = None + soname = needed = machine = build_id = None if analyze_elf and elf.is_elf(data[:4]): dynamic = elf.read_dynamic(data) - soname, needed, machine = ( + soname, needed, machine, build_id = ( dynamic.soname, dynamic.needed, dynamic.machine, + dynamic.build_id, ) entries.append( Entry( @@ -156,6 +168,7 @@ def scan(wheel_path: Path, analyze_elf: bool = False) -> WheelInfo: soname=soname, needed=needed or (), machine=machine, + build_id=build_id, head=data[:4096], ) ) @@ -221,7 +234,12 @@ def read_record(wheel_path: Path, dist_info: str) -> dict[str, tuple[str, str]]: lines = archive.read(f"{dist_info}/RECORD").decode("utf-8").splitlines() entries = (RecordEntry.from_elements(*row) for row in parse_record_file(lines)) return { - entry.path: (str(entry.hash_) if entry.hash_ else "", str(entry.size or "")) + # `entry.size or ""` would erase a legitimate 0: py.typed and namespace + # __init__.py are empty, and RECORD states 0 for them. + entry.path: ( + str(entry.hash_) if entry.hash_ else "", + "" if entry.size is None else str(entry.size), + ) for entry in entries } diff --git a/src/core/AGENTS.md b/src/core/AGENTS.md index 91bce5d657..9fd78c1edf 100644 --- a/src/core/AGENTS.md +++ b/src/core/AGENTS.md @@ -22,6 +22,8 @@ If work under **`src/core/`** went wrong—**user** correction, **pre-commit/CI* - **All C++ diagnostics go through `isaaccapture::Logger`** (`log_bridge/cpp/inc/log_bridge/logger.hpp`), never `std::cout`/`std::cerr`/`printf`, and the logger name is always `isaaccapture..`. The env-var contract and wire format are shared with the Python half, so a change to either must land in both — see [`log_bridge/AGENTS.md`](log_bridge/AGENTS.md) before touching the logging machinery, and the repo root [`AGENTS.md`](../../AGENTS.md) "Logging" section for the rules that apply to every call site. - **Run `clang-format` on touched C++ before pushing** — CI rejects unformatted C++ and pre-commit does not catch it. See the formatting instructions in the repo root [`AGENTS.md`](../../AGENTS.md) ("Pre-commit — match CI before you stop") for the exact commands; the source of truth lives there, not here. - **Authored `.py` under `src/python/isaaccapture/` carries no build-system registration.** The `CONFIGURE_DEPENDS` glob in ``src/python/CMakeLists.txt`` stages every `.py` there and `[tool.setuptools.packages.find]` discovers the packages, so a subpackage is a directory with an `__init__.py` and nothing else — no packages list, no `DEPENDS` entry. Targets that *produce* files in the staging tree (pybind ``.so``/``.pyd``, ``isaacteleop_python`` copies, ``stage_generated_tracker_exports``, ``viz_py``, vendored runtimes, …) must be **dependencies of** ``python_package`` (``DEPENDS`` / ``add_dependencies(python_package …)`` in ``src/core/python/CMakeLists.txt``), not the other way around — otherwise Windows ninja races the wheel build and setuptools fails with ``error: package directory 'isaaccapture\\' does not exist``. **``python_package`` waits for ``viz_py`` when ``BUILD_VIZ=ON``** (``.pyd`` link race). **Central stubgen** in ``src/core/python/CMakeLists.txt`` (not a ``viz_py`` POST_BUILD) DEPENDS on ``python_package``, so ``stage_generated_tracker_exports`` must stay on ``python_package``'s DEPENDS — otherwise stubbing ``isaaccapture.viz._viz`` (or any module that loads ``isaaccapture/__init__.py``) fails on the missing export. **For ``pip install -e .`` (scikit-build-core), ``_generated_tracker_exports.py`` must also be ``install(FILES ...)`` under ``isaacteleop_wheel``** — the directory install excludes ``*.py`` so authored sources stay redirect-resolved, but that module is not under ``src/python/``. +- **The staging tree (`build/python_package/$`) is a copy destination, never the only place a file exists.** A build step that writes straight into it leaves a wheel member with no build-tree origin, and the SBOM collector cannot explain it — it deliberately does not index staging, because indexing it would let the wheel explain itself. Generate into `build/generated//` and copy in, the way `stage_generated_tracker_exports` and the stub target both do. + - **Editing a `.fbs` under `schema/fbs/` can make every recording already on disk unreadable.** Recorded MCAP embeds the schema it was written under; `schema_tests` compares the current schemas against `schema/golden/` and `McapTrackerViewers` re-checks at replay time. Read [`schema/AGENTS.md`](schema/AGENTS.md) and [`schema/README.md`](schema/README.md) before changing one. - **Schema-based tracker sources exist only in a configured build tree.** They are generated at CMake *configure* time from `deviceio_trackers/trackers.toml` into `${CMAKE_BINARY_DIR}/generated/trackers/`, so grepping `src/` for a tracker such as `Se3Tracker` finds the manifest entry and the generator, not a `.cpp`. Searching, debugging, and clangd all need `cmake -B build` to have run first (already true for the `flatc` output). See [`deviceio_trackers/AGENTS.md`](deviceio_trackers/AGENTS.md) and [`codegen/AGENTS.md`](codegen/AGENTS.md) before adding or editing a tracker. - **`OpenXRSession` waits for the headset by default** (`wait_for_system = true`). `xrGetSystem` answers `XR_ERROR_FORM_FACTOR_UNAVAILABLE` (-35) until a system exists, and the retry loop is unbounded, holds the GIL for the whole wait (the pybind constructor releases nothing), and defers `Ctrl-C` until a headset turns up. Anything that constructs a session where no client will ever connect — CI, a headless smoke test — must impose its own timeout; `deps/cloudxr/docker-compose.test.yaml` wraps the CloudXR GPU tests in one for that reason. diff --git a/tests/python/sbom/CMakeLists.txt b/tests/python/sbom/CMakeLists.txt index 4403e6fc40..0f637f3d43 100644 --- a/tests/python/sbom/CMakeLists.txt +++ b/tests/python/sbom/CMakeLists.txt @@ -26,6 +26,6 @@ foreach(test_file ${TEST_FILES}) # network; and this build tree, so the file API reader is exercised against # a codemodel CMake actually wrote. set_tests_properties("sbom_${test_name}" PROPERTIES - ENVIRONMENT "ISAACTELEOP_SBOM_LICENSE_DATA=${SPDX_LICENSE_DATA_JSON_DIR};ISAACTELEOP_SBOM_BUILD_DIR=${CMAKE_BINARY_DIR}" + ENVIRONMENT "ISAACCAPTURE_SBOM_LICENSE_DATA=${SPDX_LICENSE_DATA_JSON_DIR};ISAACCAPTURE_SBOM_BUILD_DIR=${CMAKE_BINARY_DIR}" ) endforeach() diff --git a/tests/python/sbom/conftest.py b/tests/python/sbom/conftest.py index 8a67ddef8d..25fac0d0e9 100644 --- a/tests/python/sbom/conftest.py +++ b/tests/python/sbom/conftest.py @@ -32,8 +32,8 @@ def _load(name: str, path: Path): "repo_paths", Path(__file__).resolve().parents[1] / "repo_paths.py" ).repo_root -LICENSE_DATA_ENV = "ISAACTELEOP_SBOM_LICENSE_DATA" -BUILD_DIR_ENV = "ISAACTELEOP_SBOM_BUILD_DIR" +LICENSE_DATA_ENV = "ISAACCAPTURE_SBOM_LICENSE_DATA" +BUILD_DIR_ENV = "ISAACCAPTURE_SBOM_BUILD_DIR" @pytest.fixture(scope="session") diff --git a/tests/python/sbom/synth.py b/tests/python/sbom/synth.py index fd1e96a417..bf722dcfaf 100644 --- a/tests/python/sbom/synth.py +++ b/tests/python/sbom/synth.py @@ -337,10 +337,14 @@ def create(root: Path) -> Workspace: _write(build / "lib/libgamma_renamed.so", gamma_bytes) _write(build / "lib/libdelta.a", b"!\ndelta\n") _write(build / "generated/_generated.py", _GENERATED) + # Empty, like the real PEP 561 marker: RECORD states size 0 for it, and the + # bytes explain nothing, so it also stands in for any zero-length member. + _write(build / "generated/stubs/isaaccapture/py.typed", b"") # Staged tree: what the wheel is built from. _write(workspace.staged / "isaaccapture/__init__.py", _AUTHORED_INIT) _write(workspace.staged / "isaaccapture/_generated.py", _GENERATED) + _write(workspace.staged / "isaaccapture/py.typed", b"") _write(workspace.staged / "isaaccapture/viz/libgamma_renamed.so", gamma_bytes) _write(workspace.staged / "isaaccapture/viz/GAMMA_LICENSE", spdx_text("Zlib")) _write( @@ -460,6 +464,18 @@ def _file_api_reply(root: Path, build: Path, extension: str) -> None: ) +AUDITWHEEL_SBOM = "auditwheel.cdx.json" +AUDITWHEEL_CDX = json.dumps( + { + "bomFormat": "CycloneDX", + "specVersion": "1.4", + "version": 1, + "metadata": {"component": {"type": "library", "name": "isaaccapture"}}, + "components": [], + } +).encode("utf-8") + + def wheel_payload(workspace: Workspace) -> dict[str, bytes]: """The wheel setuptools would produce from the staged tree.""" payload: dict[str, bytes] = {} @@ -475,6 +491,10 @@ def wheel_payload(workspace: Workspace) -> dict[str, bytes]: workspace.root / "LICENSE.md" ).read_bytes() payload[f"{DIST_INFO}/top_level.txt"] = b"isaaccapture\nvendorpy\n" + # auditwheel runs before the collector and writes its own PEP 770 document + # for the libraries it vendored. Every fixture carries it so the suite works + # on the shape a released wheel actually has. + payload[f"{DIST_INFO}/sboms/{AUDITWHEEL_SBOM}"] = AUDITWHEEL_CDX return payload diff --git a/tests/python/sbom/test_gates.py b/tests/python/sbom/test_gates.py index 9bfdd36c5c..d8b93e673c 100644 --- a/tests/python/sbom/test_gates.py +++ b/tests/python/sbom/test_gates.py @@ -151,6 +151,16 @@ def _replace_member(wheel, name, data): archive.writestr(item, payload[item]) +def _drop_member(wheel, name): + with zipfile.ZipFile(wheel) as archive: + payload = { + item: archive.read(item) for item in archive.namelist() if item != name + } + with zipfile.ZipFile(wheel, "w", zipfile.ZIP_DEFLATED) as archive: + for item in sorted(payload): + archive.writestr(item, payload[item]) + + def test_editing_a_file_after_the_fact_fails_verification(built): _replace_member(built["wheel"], "isaaccapture/__init__.py", b"# tampered\n") @@ -180,12 +190,30 @@ def test_mutating_the_wheel_breaks_its_manifest_binding(built): assert any("modified after its digest was bound" in failure for failure in failures) -def test_a_second_document_under_sboms_is_rejected(built): - _replace_member(built["wheel"], f"{DIST_INFO}/sboms/extra.spdx.json", b"{}") +def test_an_empty_member_verifies(built): + """RECORD states size 0 for py.typed and namespace __init__.py.""" + with zipfile.ZipFile(built["wheel"]) as archive: + empty = [item.filename for item in archive.infolist() if item.file_size == 0] + + assert empty, "fixture no longer carries an empty member" + assert validate_module.check(built["wheel"]) == [] + + +def test_another_tools_document_beside_ours_is_kept(built): + """auditwheel ships `sboms/auditwheel.cdx.json`; PEP 770 shares that directory.""" + with zipfile.ZipFile(built["wheel"]) as archive: + names = set(archive.namelist()) + + assert f"{DIST_INFO}/sboms/{synth.AUDITWHEEL_SBOM}" in names + assert validate_module.check(built["wheel"]) == [] + + +def test_our_own_document_going_missing_is_rejected(built): + _drop_member(built["wheel"], f"{DIST_INFO}/sboms/{WHEEL_NAME}.spdx.json") failures = validate_module.check(built["wheel"]) - assert any("sboms/" in failure for failure in failures) + assert any("embedded SBOM" in failure for failure in failures) def test_duplicate_wheel_filenames_cannot_be_merged(tmp_path, built): diff --git a/tests/python/sbom/test_wheel_sbom.py b/tests/python/sbom/test_wheel_sbom.py index a9a2734ccb..385ea74831 100644 --- a/tests/python/sbom/test_wheel_sbom.py +++ b/tests/python/sbom/test_wheel_sbom.py @@ -40,10 +40,14 @@ def test_release_validation_passes(built): assert failures == [] -def test_sbom_lands_in_dist_info_sboms_and_nothing_else_does(built): +def test_sbom_lands_in_dist_info_sboms_beside_any_other_tools(built): sboms = {name for name in _names(built["wheel"]) if f"{DIST_INFO}/sboms/" in name} - assert sboms == {f"{DIST_INFO}/sboms/{WHEEL_NAME}.spdx.json"} + # PEP 770 shares the directory: auditwheel wrote its document before us. + assert sboms == { + f"{DIST_INFO}/sboms/{WHEEL_NAME}.spdx.json", + f"{DIST_INFO}/sboms/{synth.AUDITWHEEL_SBOM}", + } def test_every_wheel_file_is_covered(built): From 08f095bcf7a15abe20d2950f108260efbcea3d68 Mon Sep 17 00:00:00 2001 From: Andrei Aristarkhov Date: Wed, 30 Sep 2026 19:49:33 -0700 Subject: [PATCH 006/101] build(sbom): read a build-id without loading the library Indexing the machine's libraries by build-id read each one whole: 19s and 2.4GB of resident memory, because libcuda and friends are hundreds of megabytes. pyelftools seeks, so only the headers and the note need to be read -- 4s and 30MB. Empty files are excluded from the staged-tree ownership inference for the same reason they are excluded from attribution: a CMake scaffolding directory full of stamp files would otherwise adopt whichever component shipped an empty file. That alone was claiming 22 directories; three remain, all correct. A file the build transformed out of a component's tree is a derivative work carrying that component's terms, so it is labelled derived rather than generated. Signed-off-by: Andrei Aristarkhov --- scripts/sbom/sbom/discovery.py | 14 ++++-------- scripts/sbom/sbom/elf.py | 39 ++++++++++++++++++++++++---------- scripts/sbom/sbom/evidence.py | 9 +++++++- scripts/sbom/sbom/inventory.py | 15 ++++++------- scripts/sbom/sbom/licensing.py | 8 +++---- 5 files changed, 51 insertions(+), 34 deletions(-) diff --git a/scripts/sbom/sbom/discovery.py b/scripts/sbom/sbom/discovery.py index e45291a14b..ef4c11d84c 100644 --- a/scripts/sbom/sbom/discovery.py +++ b/scripts/sbom/sbom/discovery.py @@ -56,6 +56,9 @@ _DEPS_SOURCE = re.compile(r"_deps/(?P[A-Za-z0-9_.+-]+)-src(?:/|$)") +EMPTY_SHA256 = hashlib.sha256(b"").hexdigest() + + def sha256_bytes(data: bytes) -> str: return hashlib.sha256(data).hexdigest() @@ -594,16 +597,7 @@ def _host_build_id_index() -> dict[str, str]: """ index: dict[str, str] = {} for path in _host_library_paths(): - try: - data = path.read_bytes() - except OSError: - continue - if not elf.is_elf(data[:4]): - continue - try: - build_id = elf.read_dynamic(data).build_id - except Exception: # noqa: BLE001 - a malformed host library is not ours to fix - continue + build_id = elf.read_build_id(path) if not build_id: continue # A regular file beats a symlink to it: dpkg-query knows the real path. diff --git a/scripts/sbom/sbom/elf.py b/scripts/sbom/sbom/elf.py index 4a23343e7e..ffbeaaafe4 100644 --- a/scripts/sbom/sbom/elf.py +++ b/scripts/sbom/sbom/elf.py @@ -16,6 +16,7 @@ import io from dataclasses import dataclass +from pathlib import Path from elftools.elf.elffile import ELFFile @@ -44,6 +45,32 @@ def is_elf(data: bytes) -> bool: return data[:4] == ELF_MAGIC +def _build_id_from(elffile: ELFFile) -> str | None: + # Segment, not section: a stripped library keeps its program headers. + for segment in elffile.iter_segments(type="PT_NOTE"): + for note in segment.iter_notes(): + if note["n_type"] == "NT_GNU_BUILD_ID": + return note["n_desc"] + return None + + +def read_build_id(path: Path) -> str | None: + """The build-id of a file on disk, read without loading it. + + Scanning a machine's libraries means touching hundreds of files, some of + them hundreds of megabytes. pyelftools seeks, so only the headers and the + note are ever read. + """ + try: + with path.open("rb") as handle: + if handle.read(4) != ELF_MAGIC: + return None + handle.seek(0) + return _build_id_from(ELFFile(handle)) + except (OSError, Exception): # noqa: B014 - a malformed file is not ours to fix + return None + + def read_dynamic(data: bytes) -> DynamicInfo: """Parse SONAME, NEEDED and the build-id out of an in-memory ELF image.""" if not is_elf(data): @@ -65,19 +92,9 @@ def read_dynamic(data: bytes) -> DynamicInfo: elif kind == "DT_NEEDED": needed.append(tag.needed) - build_id: str | None = None - # Segment, not section, for the same reason as above. - for segment in elffile.iter_segments(type="PT_NOTE"): - for note in segment.iter_notes(): - if note["n_type"] == "NT_GNU_BUILD_ID": - build_id = note["n_desc"] - break - if build_id: - break - return DynamicInfo( soname=soname, needed=tuple(needed), machine=_ARCH.get(machine, machine), - build_id=build_id, + build_id=_build_id_from(elffile), ) diff --git a/scripts/sbom/sbom/evidence.py b/scripts/sbom/sbom/evidence.py index 1ae87dffc3..d44a767ead 100644 --- a/scripts/sbom/sbom/evidence.py +++ b/scripts/sbom/sbom/evidence.py @@ -26,6 +26,7 @@ discover_vendored, project_authors, discover_source_trees, + EMPTY_SHA256, _walk, ) @@ -187,7 +188,13 @@ def discover(repo_root: Path, build_dir: Path) -> Discovery: continue owners = set() for path in _walk(root): - match = source_files.by_hash.get(build_files.digest_of(path) or "") + digest = build_files.digest_of(path) + # Zero bytes match every other empty file, so a CMake scaffolding + # directory full of stamps would adopt whichever component happened + # to ship an empty file. They are evidence of nothing here too. + if not digest or digest == EMPTY_SHA256: + continue + match = source_files.by_hash.get(digest) if match: owners.add(match.split("/", 1)[0].removesuffix("-src")) if len(owners) == 1: diff --git a/scripts/sbom/sbom/inventory.py b/scripts/sbom/sbom/inventory.py index fe9c96ebfc..2b3906e60e 100644 --- a/scripts/sbom/sbom/inventory.py +++ b/scripts/sbom/sbom/inventory.py @@ -15,11 +15,10 @@ from pathlib import Path from typing import TYPE_CHECKING -import hashlib - from . import elf, licensing from .discovery import ( Component, + EMPTY_SHA256, resolve_system_library, resolve_system_library_by_build_id, sha256_file, @@ -88,9 +87,6 @@ def files_of(self, key: str) -> list[str]: ) -_EMPTY_SHA256 = hashlib.sha256(b"").hexdigest() - - class Resolver: """Probes a candidate's bytes and path against every discovered index.""" @@ -162,7 +158,7 @@ def resolve( # it would hand an empty marker file to whichever component happened to be # indexed first, and invent that component's license obligation with it. # The path is the only evidence such a file carries. - if candidate_digest == _EMPTY_SHA256: + if candidate_digest == EMPTY_SHA256: return self._resolve_by_path(wheel_path, source) artifacts = self._artifacts_by_hash.get(candidate_digest) @@ -211,7 +207,10 @@ def resolve( owner = self._staged_component_of(build_file) return Attribution( path=wheel_path, - origin="generated", + # Transforming a component's own file leaves a derivative work, + # which carries that component's terms. Only output that owes + # nothing to a third party is merely "generated". + origin="derived" if owner else "generated", detail=f"{source}; {build_file}", components={owner: {"derived-file"}} if owner else {}, primary=owner, @@ -378,7 +377,7 @@ def _resolve_by_path(self, wheel_path: str, source: str) -> Attribution | None: owner = self._staged_component_of(build_file) return Attribution( path=wheel_path, - origin="generated", + origin="derived" if owner else "generated", detail=f"{source}; {build_file}", components={owner: {"derived-file"}} if owner else {}, primary=owner, diff --git a/scripts/sbom/sbom/licensing.py b/scripts/sbom/sbom/licensing.py index 3da74e664a..dad49ffbd3 100644 --- a/scripts/sbom/sbom/licensing.py +++ b/scripts/sbom/sbom/licensing.py @@ -395,10 +395,10 @@ def license_ref(component_key: str) -> str: def expressible(license_id: str) -> bool: """Whether the SPDX grammar we ship can validate this identifier. - The corpus the build fetches moves ahead of what `license-expression` - knows -- SPDX 3.29.0 added BSD-2-Clause-pos-unchanged, which no released - version of the grammar recognises. Matching against the newer corpus is - right; naming an ID a consumer's validator will reject is not. + The SPDX license list the build fetches moves ahead of the list bundled in + `license-expression`, so the corpus can name an ID the grammar cannot parse. + Matching against the newer corpus is right; naming an ID that a consumer's + validator -- which ships the older list -- will reject is not. """ if license_id.startswith("LicenseRef-"): return True From d5620e9be5a8f96be9edd29f4a8bd0bbeeb38b67 Mon Sep 17 00:00:00 2001 From: Andrei Aristarkhov Date: Wed, 30 Sep 2026 19:49:34 -0700 Subject: [PATCH 007/101] docs(sbom): say which two members the document omits RECORD hashes every member including this document, and the document is rendered before RECORD is regenerated, so listing either is a fixed point no hash function has. SPDX 2.3 requires a checksum on every file listed, so they are omitted rather than listed without one. A reader diffing the file list against the wheel finds exactly two gaps and nothing explaining them. Signed-off-by: Andrei Aristarkhov --- docs/source/references/sbom.rst | 4 ++++ scripts/sbom/sbom/build.py | 5 +++++ 2 files changed, 9 insertions(+) diff --git a/docs/source/references/sbom.rst b/docs/source/references/sbom.rst index 328cf05b23..db3ee791ec 100644 --- a/docs/source/references/sbom.rst +++ b/docs/source/references/sbom.rst @@ -50,6 +50,10 @@ Its per-file digests are there so the description can be shown to belong to the front of you, and so it can be checked against ``RECORD``; they are not a second root of trust. +The document lists every member of the wheel except two: ``RECORD`` and the document +itself. Neither can carry its own digest, since ``RECORD`` hashes the document and the +document would hash ``RECORD``. ``check`` reads both against the archive directly. + .. code-block:: bash git clone --depth 1 https://github.com/NVIDIA/IsaacCapture diff --git a/scripts/sbom/sbom/build.py b/scripts/sbom/sbom/build.py index e5f7d6cf86..5d3395a2d7 100644 --- a/scripts/sbom/sbom/build.py +++ b/scripts/sbom/sbom/build.py @@ -138,6 +138,11 @@ def build( metadata_declared = patched_metadata != metadata_raw replacements = {f"{wheel.dist_info}/METADATA": patched_metadata} + # Neither file can state its own digest. RECORD hashes every member, this + # document included, and the document is rendered before RECORD is + # regenerated -- listing either is a fixed point no hash function has. SPDX + # 2.3 requires a checksum on every file listed, so they are omitted rather + # than listed without one; `check` reads both against the archive instead. excluded = [f"{wheel.dist_info}/RECORD", sbom_name] projected = [item for item in wheel.entries if item.name not in replacements] + [ From 49da7423fd316ef8bf0980c24b54a8128e4800fe Mon Sep 17 00:00:00 2001 From: Andrei Aristarkhov Date: Wed, 30 Sep 2026 19:49:35 -0700 Subject: [PATCH 008/101] build(sbom): do not walk another build tree as source Only the build directory in use was skipped when indexing the repository, so a second one -- a developer's `build-debug`, or the tree scikit-build-core makes -- was walked as if it were source. Its staged copies made a file's origin ambiguous, which surfaces as a member that reached the wheel by an unknown route: a confusing error pointing at the wrong thing. A configured tree says what it is with a CMakeCache.txt, which is what is skipped now rather than a list of names. Signed-off-by: Andrei Aristarkhov --- scripts/sbom/sbom/evidence.py | 11 ++++++++++- tests/python/sbom/test_discovery.py | 25 +++++++++++++++++++++++++ 2 files changed, 35 insertions(+), 1 deletion(-) diff --git a/scripts/sbom/sbom/evidence.py b/scripts/sbom/sbom/evidence.py index d44a767ead..9e548d90e6 100644 --- a/scripts/sbom/sbom/evidence.py +++ b/scripts/sbom/sbom/evidence.py @@ -159,10 +159,19 @@ def discover(repo_root: Path, build_dir: Path) -> Discovery: repo_files = FileIndex() # Skip the build tree by path, not by name: --build-dir is free-form, and a # differently-named one inside the repo would otherwise be walked twice. + # Any *other* configured tree is skipped too, found by the CMakeCache.txt + # that makes it one -- a second build directory holds staged copies of the + # repo, and walking them as source leaves a file's origin ambiguous rather + # than wrong, which reads as an unexplained member. + other_builds = tuple( + path.parent + for pattern in ("*/CMakeCache.txt", "*/*/CMakeCache.txt") + for path in repo_root.glob(pattern) + ) repo_files.add_tree( repo_root, ".", - skip=(build_dir, repo_root / "build-wheel", repo_root / "dist"), + skip=(build_dir, *other_builds, repo_root / "build-wheel", repo_root / "dist"), ) # _deps/*-src is the component-source domain, already indexed above; a wheel diff --git a/tests/python/sbom/test_discovery.py b/tests/python/sbom/test_discovery.py index 4dbab85130..a97bbfbc68 100644 --- a/tests/python/sbom/test_discovery.py +++ b/tests/python/sbom/test_discovery.py @@ -188,3 +188,28 @@ def test_the_codemodel_reader_works_on_a_codemodel_cmake_wrote( assert graph.by_name(output.name), "every artifact is findable by name" for how in graph.contributions(artifact).values(): assert how <= {"compiled-source", "header-include"} + + +def test_a_second_build_tree_is_not_mistaken_for_source(workspace, license_data): + """A developer's other build directory holds staged copies of the repo. + + Walking those as source makes a file's origin ambiguous, which surfaces as + an unexplained wheel member rather than a wrong one. A configured tree says + what it is with a CMakeCache.txt. + """ + from sbom import evidence, licensing + + licensing.load_corpus(license_data) + stray = workspace.root / "build-debug" + (stray / "isaaccapture").mkdir(parents=True) + (stray / "CMakeCache.txt").write_text("CMAKE_BUILD_TYPE:STRING=Debug\n") + copy = stray / "isaaccapture" / "__init__.py" + copy.write_text( + (workspace.root / "src/python/isaaccapture/__init__.py").read_text() + ) + + discovered = evidence.discover(workspace.root, workspace.build) + indexed = set(discovered.repo_files.by_hash.values()) + + assert "./src/python/isaaccapture/__init__.py" in indexed + assert not any(item.startswith("./build-debug/") for item in indexed) From a3f7e525b9d3cc666a4e3043a3934275fe522c15 Mon Sep 17 00:00:00 2001 From: Andrei Aristarkhov Date: Wed, 30 Sep 2026 19:49:37 -0700 Subject: [PATCH 009/101] build(sbom): name a supplier and stop implying a pinned version A system library's supplier read `Organization: libbsd0` -- the package name in a slot meant for whoever provided it. dpkg records a maintainer, which is the answer, formatted the way SPDX asks for it. A declared dependency's SPDXID was slugified from the whole requirement, so `websockets>=14.0` became `websockets-14.0` and read as a pinned version beside a versionInfo this document deliberately leaves unasserted. The same name recurs under different extras, so uniqueness comes from a digest of the requirement; its text was always in the comment. Signed-off-by: Andrei Aristarkhov --- scripts/sbom/sbom/discovery.py | 29 ++++++++++++++++++++++++++++- scripts/sbom/sbom/document.py | 8 +++++++- tests/python/sbom/test_gates.py | 7 ++++++- 3 files changed, 41 insertions(+), 3 deletions(-) diff --git a/scripts/sbom/sbom/discovery.py b/scripts/sbom/sbom/discovery.py index ef4c11d84c..1c3bd1d060 100644 --- a/scripts/sbom/sbom/discovery.py +++ b/scripts/sbom/sbom/discovery.py @@ -665,12 +665,39 @@ def _identify_host_package(library: Path, origin: dict) -> dict: package = result.stdout.split(":", 1)[0].strip() origin["package"] = package + origin["supplier"] = _package_supplier(package) copyright_file = Path("/usr/share/doc") / package / "copyright" if copyright_file.is_file(): origin["copyright"] = str(copyright_file) return origin +def _package_supplier(package: str) -> str | None: + """Who ships a distro package, in the form SPDX wants. + + The package name is not a supplier: `libbsd0` names the thing, not whoever + provided it. dpkg records a maintainer, which is the answer. + """ + try: + result = subprocess.run( + ["dpkg-query", "-W", "-f=${Maintainer}", package], + capture_output=True, + text=True, + check=False, + timeout=60, + ) + except (OSError, subprocess.SubprocessError): + return None + maintainer = result.stdout.strip() + if result.returncode != 0 or not maintainer: + return None + # "Name " is Debian's form; SPDX wants "Organization: Name (email)". + match = re.match(r"^(?P.+?)\s*<(?P[^>]+)>$", maintainer) + if match: + return f"Organization: {match['name']} ({match['email']})" + return f"Organization: {maintainer}" + + def _license_near(library: Path) -> Path | None: """A toolchain that installs outside the package manager still ships its terms. @@ -716,7 +743,7 @@ def system_component(soname: str, origin: dict) -> Component: key=key, name=soname, kind="system-library", - supplier=f"Organization: {package}" if package else "NOASSERTION", + supplier=origin.get("supplier") or "NOASSERTION", homepage="NOASSERTION", purl=PackageURL(type="generic", name=soname).to_string(), version="NOASSERTION", diff --git a/scripts/sbom/sbom/document.py b/scripts/sbom/sbom/document.py index 2592e0e3c2..c67afdf810 100644 --- a/scripts/sbom/sbom/document.py +++ b/scripts/sbom/sbom/document.py @@ -275,7 +275,13 @@ def build_document( for requirement in inventory.requires_dist: name = Requirement(requirement).name - package_id = _spdx_id("Package-pypi", requirement) + # Not the requirement string: slugifying `websockets>=14.0` yields + # `websockets-14.0`, which reads as a pinned version beside a + # versionInfo this document deliberately leaves unasserted. The same + # name recurs under different extras, so uniqueness comes from a digest + # of the whole requirement; its text is in the comment below. + digest = hashlib.sha256(requirement.encode("utf-8")).hexdigest()[:8] + package_id = _spdx_id("Package-pypi", f"{name}-{digest}") packages.append( { "SPDXID": package_id, diff --git a/tests/python/sbom/test_gates.py b/tests/python/sbom/test_gates.py index d8b93e673c..dec6f1c202 100644 --- a/tests/python/sbom/test_gates.py +++ b/tests/python/sbom/test_gates.py @@ -126,6 +126,9 @@ def test_a_vendored_library_is_attributed_to_the_package_that_supplied_it( "soname": soname, "resolved_path": "/usr/lib/libz.so.1.3", "package": "zlib1g", + # dpkg's maintainer, not the package name: `zlib1g` names the thing, + # not whoever supplied it. + "supplier": "Organization: Ubuntu Developers (ubuntu-devel@example.com)", "copyright": str(copyright_file), }, ) @@ -139,7 +142,9 @@ def test_a_vendored_library_is_attributed_to_the_package_that_supplied_it( item for item in report["components"] if item["component"] == "system:libz.so.1" ) assert vendored["license_concluded"] == "Zlib" - assert vendored["supplier"] == "Organization: zlib1g" + assert vendored["supplier"] == ( + "Organization: Ubuntu Developers (ubuntu-devel@example.com)" + ) def _replace_member(wheel, name, data): From 180164c7ab03df2b56711d81dc7132dfb5d019a0 Mon Sep 17 00:00:00 2001 From: Andrei Aristarkhov Date: Wed, 30 Sep 2026 19:49:38 -0700 Subject: [PATCH 010/101] build(sbom): record where a fetched archive came from, and a range as a range The CloudXR tarballs are downloaded at configure time, not checked in, so the only evidence of their origin was a shell variable that stopped existing when the script exited. The fetch script now writes the URL it used beside the tarball, and the collector reads it: download location, and the host that served it as supplier. A `Requires-Dist` constraint is a range, which purl cannot express -- its version field holds one exact version. VERS can, so the specifier is carried as a structured external reference rather than only in a sentence, and a requirement gated on an extra is an OPTIONAL_DEPENDENCY_OF rather than a flat dependency. A name match that a build-id contradicts is refused. Two SDKs ship a library called libcloudxr.so; claiming one is the other would hand it the wrong component's licence. Signed-off-by: Andrei Aristarkhov --- deps/cloudxr/.gitignore | 2 + scripts/download_cloudxr_runtime_sdk.sh | 5 ++ scripts/sbom/sbom/discovery.py | 8 ++- scripts/sbom/sbom/document.py | 77 +++++++++++++++++++------ scripts/sbom/sbom/evidence.py | 38 ++++++++++-- scripts/sbom/sbom/inventory.py | 29 ++++++++-- tests/python/sbom/synth.py | 46 ++++++++++++--- tests/python/sbom/test_discovery.py | 28 +++++++++ tests/python/sbom/test_gates.py | 29 ++++++++++ 9 files changed, 225 insertions(+), 37 deletions(-) diff --git a/deps/cloudxr/.gitignore b/deps/cloudxr/.gitignore index 78883dbf52..5897b633ff 100644 --- a/deps/cloudxr/.gitignore +++ b/deps/cloudxr/.gitignore @@ -1,5 +1,7 @@ # CloudXR runtime and SDK tarballs *.tar.gz +# Where the download came from, written beside it by the fetch script. +*.tar.gz.source *.tgz # CloudXR environment files diff --git a/scripts/download_cloudxr_runtime_sdk.sh b/scripts/download_cloudxr_runtime_sdk.sh index caa643308f..d45e91649c 100755 --- a/scripts/download_cloudxr_runtime_sdk.sh +++ b/scripts/download_cloudxr_runtime_sdk.sh @@ -112,6 +112,11 @@ download_ngc_file() { rm -f "$out_path" return 1 fi + # Record where this came from. The tarball is fetched, not checked in, so + # without this the only evidence of its origin is a shell variable that + # stops existing when the script exits -- and the SBOM has to call the + # download location unknown. + printf '%s\n' "$url" > "${out_path}.source" } # Try each remote name in turn; the first that resolves wins. diff --git a/scripts/sbom/sbom/discovery.py b/scripts/sbom/sbom/discovery.py index 1c3bd1d060..60c123923a 100644 --- a/scripts/sbom/sbom/discovery.py +++ b/scripts/sbom/sbom/discovery.py @@ -27,7 +27,7 @@ import tomllib import tarfile import zipfile -from dataclasses import dataclass +from dataclasses import dataclass, replace from pathlib import Path from packageurl import PackageURL @@ -768,6 +768,7 @@ class Member: path: str sha256: str size: int + build_id: str | None = None class ArchiveIndex: @@ -794,15 +795,16 @@ def add_archive(self, path: Path, display: str) -> None: "member_count": len(members), } for member, data in members: - self.by_hash.setdefault(member.sha256, member) - self.by_name.setdefault(Path(member.path).name, []).append(member) if elf.is_elf(data[:4]): try: build_id = elf.read_dynamic(data).build_id except Exception: # noqa: BLE001 - a malformed member is not ours to fix build_id = None if build_id: + member = replace(member, build_id=build_id) self.by_build_id.setdefault(build_id, member) + self.by_hash.setdefault(member.sha256, member) + self.by_name.setdefault(Path(member.path).name, []).append(member) self._maybe_license(display, member, data) def _maybe_license(self, display: str, member: Member, data: bytes) -> None: diff --git a/scripts/sbom/sbom/document.py b/scripts/sbom/sbom/document.py index c67afdf810..7d917bf9f4 100644 --- a/scripts/sbom/sbom/document.py +++ b/scripts/sbom/sbom/document.py @@ -6,6 +6,7 @@ from __future__ import annotations import hashlib +import re import os from datetime import datetime, timezone @@ -112,6 +113,43 @@ def _extracted_licenses(components: list[Component]) -> list[dict]: return [entries[key] for key in sorted(entries)] +_EXTRA_MARKER = re.compile(r"\bextra\s*==") + + +def _is_extra_gated(requirement: Requirement) -> bool: + return requirement.marker is not None and bool( + _EXTRA_MARKER.search(str(requirement.marker)) + ) + + +def _dependency_refs(name: str, requirement: Requirement) -> list[dict]: + """purl for the package, and the constraint as a range rather than prose. + + purl carries one exact version, which a requirement does not have. VERS (the + purl project's range notation) does, so the specifier stays machine-readable + instead of living only in a sentence. + """ + refs = [ + { + "referenceCategory": "PACKAGE-MANAGER", + "referenceType": "purl", + "referenceLocator": PackageURL(type="pypi", name=name).to_string(), + } + ] + constraints = sorted( + f"{item.operator}{item.version}" for item in requirement.specifier + ) + if constraints: + refs.append( + { + "referenceCategory": "OTHER", + "referenceType": "vers", + "referenceLocator": "vers:pypi/" + "|".join(constraints), + } + ) + return refs + + def build_document( project: dict, evidence_doc: dict, @@ -274,7 +312,8 @@ def build_document( ) for requirement in inventory.requires_dist: - name = Requirement(requirement).name + parsed = Requirement(requirement) + name = parsed.name # Not the requirement string: slugifying `websockets>=14.0` yields # `websockets-14.0`, which reads as a pinned version beside a # versionInfo this document deliberately leaves unasserted. The same @@ -293,28 +332,32 @@ def build_document( "licenseConcluded": NOASSERTION, "licenseDeclared": NOASSERTION, "copyrightText": NOASSERTION, - "externalRefs": [ - { - "referenceCategory": "PACKAGE-MANAGER", - "referenceType": "purl", - "referenceLocator": PackageURL( - type="pypi", name=name - ).to_string(), - } - ], + "externalRefs": _dependency_refs(name, parsed), "comment": ( f"Consumer requirement declared in wheel metadata: {requirement}. " "Resolved at install time; this document states no license for it." ), } ) - relationships.append( - { - "spdxElementId": wheel_id, - "relatedSpdxElement": package_id, - "relationshipType": "DEPENDS_ON", - } - ) + # An `extra ==` marker means the requirement only applies when that extra + # is requested, which SPDX has a relationship for. Its direction is the + # reverse of DEPENDS_ON. + if _is_extra_gated(parsed): + relationships.append( + { + "spdxElementId": package_id, + "relatedSpdxElement": wheel_id, + "relationshipType": "OPTIONAL_DEPENDENCY_OF", + } + ) + else: + relationships.append( + { + "spdxElementId": wheel_id, + "relatedSpdxElement": package_id, + "relationshipType": "DEPENDS_ON", + } + ) # SOURCE_DATE_EPOCH makes the whole wheel reproducible, which is what lets a # rebuild be compared against a published one byte for byte. diff --git a/scripts/sbom/sbom/evidence.py b/scripts/sbom/sbom/evidence.py index 9e548d90e6..7a1eb98284 100644 --- a/scripts/sbom/sbom/evidence.py +++ b/scripts/sbom/sbom/evidence.py @@ -10,6 +10,7 @@ from dataclasses import dataclass from datetime import datetime, timezone from pathlib import Path +from urllib.parse import urlparse from packageurl import PackageURL @@ -83,7 +84,30 @@ def _cmake_cache(build_dir: Path) -> dict[str, str]: return entries -def _archive_components(archives: ArchiveIndex) -> dict[str, Component]: +def _fetched_from(archive: Path) -> str | None: + """The URL a fetch script recorded beside an archive it downloaded. + + Most archives here are downloaded at configure time rather than checked in, + so the origin is only knowable if whatever fetched it wrote the URL down. + """ + marker = archive.with_name(archive.name + ".source") + if not marker.is_file(): + return None + url = marker.read_text(encoding="utf-8", errors="replace").strip() + return url.splitlines()[0].strip() if url else None + + +def _supplier_of(url: str | None) -> str: + """Whoever served the archive, named by the host that did.""" + if not url: + return "NOASSERTION" + host = urlparse(url).hostname + return f"Organization: {host}" if host else "NOASSERTION" + + +def _archive_components( + archives: ArchiveIndex, repo_root: Path +) -> dict[str, Component]: """A tarball in the repo is a component: its members ship as they are.""" components: dict[str, Component] = {} for display, record in archives.archives.items(): @@ -103,18 +127,22 @@ def _archive_components(archives: ArchiveIndex) -> dict[str, Component]: ] concluded, declared = licensing.expression(evidence, name) + source_url = _fetched_from(repo_root / display) + held = ( + f"fetched from {source_url}" if source_url else "present in the source tree" + ) components[display] = Component( key=display, name=name, kind="archive", - supplier="NOASSERTION", + supplier=_supplier_of(source_url), homepage="NOASSERTION", purl=PackageURL(type="generic", name=name).to_string(), version=_archive_version(archives, display) or "NOASSERTION", - download_location="NOASSERTION", + download_location=source_url or "NOASSERTION", source_info=( f"Unpacked from {display} (sha256:{record['sha256']}, " - f"{record['member_count']} members) held in this repository." + f"{record['member_count']} members), {held}." ), license_concluded=concluded, license_declared=declared, @@ -149,7 +177,7 @@ def discover(repo_root: Path, build_dir: Path) -> Discovery: archives = discover_archives( repo_root, skip=(build_dir, repo_root / "build-wheel", repo_root / "dist") ) - components.update(_archive_components(archives)) + components.update(_archive_components(archives, repo_root)) source_files = FileIndex() for root in sorted(deps_dir.glob("*-src")): diff --git a/scripts/sbom/sbom/inventory.py b/scripts/sbom/sbom/inventory.py index 2b3906e60e..0ae2a4991d 100644 --- a/scripts/sbom/sbom/inventory.py +++ b/scripts/sbom/sbom/inventory.py @@ -159,7 +159,7 @@ def resolve( # indexed first, and invent that component's license obligation with it. # The path is the only evidence such a file carries. if candidate_digest == EMPTY_SHA256: - return self._resolve_by_path(wheel_path, source) + return self._resolve_by_path(wheel_path, source, build_id) artifacts = self._artifacts_by_hash.get(candidate_digest) if artifacts: @@ -224,7 +224,7 @@ def resolve( if by_build_id is not None: return by_build_id - by_path = self._resolve_by_path(wheel_path, source) + by_path = self._resolve_by_path(wheel_path, source, build_id) if by_path is not None: return by_path @@ -323,16 +323,26 @@ def _from_build_host_by_build_id( primary=component.key, ) - def _resolve_by_path(self, wheel_path: str, source: str) -> Attribution | None: + def _resolve_by_path( + self, wheel_path: str, source: str, build_id: str | None = None + ) -> Attribution | None: """Post-processing changes bytes; the path still says where they came from. patchelf, auditwheel's RPATH rewrite and the MJCF mesh stripping all leave a file that no longer hashes to its origin, so a unique path match is the remaining evidence -- recorded as derived, never as an exact copy. + + A name match is only ever a guess, so a build-id that disagrees overrules + it: two libraries called libcloudxr.so from different SDKs are not each + other, and claiming they are would hand one the other's licence. """ name = Path(wheel_path).name - members = self.archives.by_name.get(name, []) + members = [ + member + for member in self.archives.by_name.get(name, []) + if not _build_ids_disagree(build_id, member.build_id) + ] containers = {member.container for member in members} if len(containers) == 1: member = members[0] @@ -347,7 +357,11 @@ def _resolve_by_path(self, wheel_path: str, source: str) -> Attribution | None: primary=member.container, ) - artifacts = self.graph.by_name(name) + artifacts = [ + artifact + for artifact in self.graph.by_name(name) + if not _build_ids_disagree(build_id, _build_id_of(artifact.output)) + ] if len(artifacts) == 1: return self._from_artifact( artifacts[0], wheel_path, f"{source}; modified build output" @@ -441,6 +455,11 @@ def build( ) +def _build_ids_disagree(left: str | None, right: str | None) -> bool: + """True only when both are known and differ -- that is evidence, not absence.""" + return bool(left and right and left != right) + + def _build_id_of(path: Path) -> str | None: """The build-id of a file on disk, or None if it is not an ELF.""" try: diff --git a/tests/python/sbom/synth.py b/tests/python/sbom/synth.py index bf722dcfaf..783724c30b 100644 --- a/tests/python/sbom/synth.py +++ b/tests/python/sbom/synth.py @@ -32,6 +32,7 @@ WHEEL_NAME = "isaaccapture-0.4.0-cp312-cp312-manylinux_2_35_x86_64.whl" DIST_INFO = "isaaccapture-0.4.0.dist-info" SDK_ARCHIVE = "deps/vendor/demosdk-1.2.0-linux-amd64.tar.gz" +SDK_DEMO_BUILD_ID = "1111111111111111111111111111111111111111" # ============================================================================== # ELF @@ -39,6 +40,8 @@ _PT_LOAD = 1 _PT_DYNAMIC = 2 +_PT_NOTE = 4 +_NT_GNU_BUILD_ID = 3 _DT_NULL = 0 _DT_NEEDED = 1 _DT_STRTAB = 5 @@ -50,10 +53,30 @@ _BASE_VADDR = 0x400000 +def _build_id_note(build_id: str) -> bytes: + """A .note.gnu.build-id in the form the linker writes it.""" + name = b"GNU\0" + desc = bytes.fromhex(build_id) + padding = b"\0" * (-len(desc) % 4) + return ( + struct.pack(" bytes: - """A 64-bit little-endian ELF carrying just a readable dynamic segment.""" + """A 64-bit little-endian ELF carrying a readable dynamic segment. + + With `build_id`, it also carries the note the linker writes -- which is what + survives a repair tool rewriting SONAME and RPATH. + """ strings = [b"\0"] # index 0 is the empty string, per the ELF string-table format offsets: dict[str, int] = {} cursor = 1 @@ -69,11 +92,14 @@ def elf_shared_object( entries = [(_DT_SONAME, offsets[soname])] entries += [(_DT_NEEDED, offsets[name]) for name in needed] - header_size = _EHDR_SIZE + 2 * _PHDR_SIZE + note = _build_id_note(build_id) if build_id else b"" + segments = 3 if note else 2 + header_size = _EHDR_SIZE + segments * _PHDR_SIZE dynamic_offset = header_size dynamic_size = (len(entries) + 3) * 16 strtab_offset = dynamic_offset + dynamic_size - total = strtab_offset + len(strtab) + len(padding) + note_offset = strtab_offset + len(strtab) + total = note_offset + len(note) + len(padding) entries.append((_DT_STRTAB, _BASE_VADDR + strtab_offset)) entries.append((_DT_STRSZ, len(strtab))) @@ -90,7 +116,7 @@ def elf_shared_object( struct.pack_into(" None: base = _EHDR_SIZE + index * _PHDR_SIZE @@ -104,12 +130,16 @@ def phdr(index: int, kind: int, offset: int, size: int) -> None: phdr(0, _PT_LOAD, 0, total) phdr(1, _PT_DYNAMIC, dynamic_offset, len(entries) * 16) + if note: + phdr(2, _PT_NOTE, note_offset, len(note)) for index, (tag, value) in enumerate(entries): struct.pack_into(" dict[str, bytes]: members = { "LICENSE.txt": PROPRIETARY_LICENSE.encode("utf-8"), "VERSION": b"1.2.0\n", - "libdemo.so": elf_shared_object("libdemo.so", ("libc.so.6",)), + "libdemo.so": elf_shared_object( + "libdemo.so", ("libc.so.6",), build_id=SDK_DEMO_BUILD_ID + ), "libextra.so": elf_shared_object("libextra.so", ("libssl.so.3", "libc.so.6")), "include/demo.h": b"#pragma once\n", } diff --git a/tests/python/sbom/test_discovery.py b/tests/python/sbom/test_discovery.py index a97bbfbc68..99e6be7d41 100644 --- a/tests/python/sbom/test_discovery.py +++ b/tests/python/sbom/test_discovery.py @@ -213,3 +213,31 @@ def test_a_second_build_tree_is_not_mistaken_for_source(workspace, license_data) assert "./src/python/isaaccapture/__init__.py" in indexed assert not any(item.startswith("./build-debug/") for item in indexed) + + +def test_a_fetched_archive_records_where_it_came_from(workspace, license_data): + """Archives here are downloaded at configure time, not checked in. + + The fetch script writes the URL beside the tarball, so the origin is read + rather than guessed; without that file the document must say it does not + know. + """ + from sbom import evidence, licensing + + licensing.load_corpus(license_data) + archive = workspace.root / synth.SDK_ARCHIVE + + unknown = evidence.discover(workspace.root, workspace.build) + before = unknown.components[synth.SDK_ARCHIVE] + assert before.download_location == "NOASSERTION" + assert before.supplier == "NOASSERTION" + + archive.with_name(archive.name + ".source").write_text( + "https://api.ngc.nvidia.com/v2/resources/example/sdk.tar.gz\n" + ) + known = evidence.discover(workspace.root, workspace.build) + after = known.components[synth.SDK_ARCHIVE] + + assert after.download_location.startswith("https://api.ngc.nvidia.com/") + assert after.supplier == "Organization: api.ngc.nvidia.com" + assert "fetched from https://" in after.source_info diff --git a/tests/python/sbom/test_gates.py b/tests/python/sbom/test_gates.py index dec6f1c202..3581eb3e27 100644 --- a/tests/python/sbom/test_gates.py +++ b/tests/python/sbom/test_gates.py @@ -314,3 +314,32 @@ def test_building_twice_on_the_same_wheel_is_refused(built): built["wheel"], built["out_dir"], ) + + +def test_a_name_match_a_build_id_contradicts_is_refused(workspace, license_data): + """Two SDKs can ship a library under the same file name. + + The patched copy no longer hashes to either, so only the name is left to go + on -- and a differing build-id says it is the wrong one. Claiming it would + hand this binary the other component's licence. + """ + from sbom import licensing + + licensing.load_corpus(license_data) + payload = synth.wheel_payload(workspace) + # Same basename as the member inside the SDK archive, different build-id, + # and patched so the content matches nothing. + payload["isaaccapture/other_sdk/libdemo.so"] = synth.elf_shared_object( + "libdemo.so", + ("libc.so.6", "libm.so.6"), + build_id="2222222222222222222222222222222222222222", + ) + synth.write_wheel(workspace.wheel, payload) + + with pytest.raises(build_module.BuildError, match="could not be traced"): + build_module.build( + workspace.root, + workspace.build, + workspace.wheel, + workspace.root / "sbom", + ) From cffb25cdcfd23293914b435ddf23c7747426dd46 Mon Sep 17 00:00:00 2001 From: Andrei Aristarkhov Date: Wed, 30 Sep 2026 19:49:40 -0700 Subject: [PATCH 011/101] build(sbom): drop the fallbacks that hid a changed condition Audited every handler and default in the collector, keeping the ones whose fallback degrades into a failure that is already loud and removing the rest. Gone: two `except ValueError` guards around `relative_to`, unreachable because the walk yields only files beneath its root; a second implementation of the build-id reader carrying its own handlers; and the swallow that let an archive this build fetched fail to open and reappear as every file it should have explained being unattributable. Narrowed: `except (OSError, Exception)` caught everything including bugs in this parser, and is now the ELF errors it meant. Made loud: a missing CMAKE_BUILD_TYPE silently became Release, pointing the build graph and the staged-tree probe at a directory that need not exist. Archives inside a build tree were indexed as if held in the repository, contradicting the docstring; the two call sites now share one list of what is not source. Signed-off-by: Andrei Aristarkhov --- scripts/sbom/sbom/cli.py | 2 ++ scripts/sbom/sbom/discovery.py | 30 +++++++++--------- scripts/sbom/sbom/elf.py | 5 ++- scripts/sbom/sbom/evidence.py | 47 +++++++++++++++++------------ scripts/sbom/sbom/inventory.py | 18 ++--------- tests/python/sbom/test_discovery.py | 19 ++++++++++++ tests/python/sbom/test_gates.py | 17 +++++++++++ 7 files changed, 86 insertions(+), 52 deletions(-) diff --git a/scripts/sbom/sbom/cli.py b/scripts/sbom/sbom/cli.py index 3ccf00e9c5..bee76a5705 100644 --- a/scripts/sbom/sbom/cli.py +++ b/scripts/sbom/sbom/cli.py @@ -13,6 +13,7 @@ from . import TOOL_NAME, TOOL_VERSION from . import build as build_module +from . import discovery from . import evidence as evidence_module from . import licensing from . import validate as validate_module @@ -182,6 +183,7 @@ def main(argv: list[str] | None = None) -> int: evidence_module.EvidenceError, build_module.BuildError, licensing.CorpusError, + discovery.ArchiveError, ) as error: print(f"error: {error}", file=sys.stderr) return 2 diff --git a/scripts/sbom/sbom/discovery.py b/scripts/sbom/sbom/discovery.py index 60c123923a..fd7b3821af 100644 --- a/scripts/sbom/sbom/discovery.py +++ b/scripts/sbom/sbom/discovery.py @@ -30,6 +30,7 @@ from dataclasses import dataclass, replace from pathlib import Path +from elftools.common.exceptions import ELFError from packageurl import PackageURL from . import elf, licensing @@ -255,6 +256,10 @@ def _owner(path: Path, ownership: Ownership | None) -> str | None: return ownership.owner_of(path) if ownership is not None else None +class ArchiveError(Exception): + """An archive the build fetched cannot be read.""" + + class FileApiError(Exception): """The build tree carries no CMake file API reply to read.""" @@ -786,8 +791,6 @@ def add_archive(self, path: Path, display: str) -> None: if path.stat().st_size > _MAX_ARCHIVE_BYTES: return members = self._read_members(path, display) - if members is None: - return self.archives[display] = { "path": display, "sha256": sha256_file(path), @@ -798,7 +801,7 @@ def add_archive(self, path: Path, display: str) -> None: if elf.is_elf(data[:4]): try: build_id = elf.read_dynamic(data).build_id - except Exception: # noqa: BLE001 - a malformed member is not ours to fix + except ELFError: # a third-party archive may hold a broken ELF build_id = None if build_id: member = replace(member, build_id=build_id) @@ -821,7 +824,7 @@ def _maybe_license(self, display: str, member: Member, data: bytes) -> None: self.archives[display].setdefault("license_members", []).append((member, data)) @staticmethod - def _read_members(path: Path, display: str) -> list[tuple[Member, bytes]] | None: + def _read_members(path: Path, display: str) -> list[tuple[Member, bytes]]: """Hash every member; keep the bytes only of the few worth keeping. An SDK tarball expands to far more than it compresses to, so holding @@ -849,8 +852,12 @@ def take(name: str, data: bytes) -> None: handle = archive.extractfile(info) if handle is not None: take(info.name.removeprefix("./"), handle.read()) - except (OSError, tarfile.TarError, zipfile.BadZipFile): - return None + except (OSError, tarfile.TarError, zipfile.BadZipFile) as error: + # Only archives this build deliberately fetched are indexed, so one + # that will not open is a broken download, not a stray file. Saying + # so beats reporting every file it should have explained as a member + # that reached the wheel by an unknown route. + raise ArchiveError(f"{display} cannot be read: {error}") from error return collected @@ -864,11 +871,7 @@ def discover_archives(repo_root: Path, skip: tuple[Path, ...] = ()) -> ArchiveIn name = path.name.lower() if not name.endswith(ARCHIVE_SUFFIXES): continue - try: - relative = path.relative_to(repo_root).as_posix() - except ValueError: - continue - index.add_archive(path, relative) + index.add_archive(path, path.relative_to(repo_root).as_posix()) return index @@ -891,10 +894,7 @@ def add_tree( for path in _walk(root): if any(str(path).startswith(str(item)) for item in skipped): continue - try: - relative = path.relative_to(root).as_posix() - except ValueError: - continue + relative = path.relative_to(root).as_posix() display = f"{display_root}/{relative}" try: digest = sha256_file(path) diff --git a/scripts/sbom/sbom/elf.py b/scripts/sbom/sbom/elf.py index ffbeaaafe4..73d95f1bda 100644 --- a/scripts/sbom/sbom/elf.py +++ b/scripts/sbom/sbom/elf.py @@ -18,6 +18,7 @@ from dataclasses import dataclass from pathlib import Path +from elftools.common.exceptions import ELFError from elftools.elf.elffile import ELFFile ELF_MAGIC = b"\x7fELF" @@ -67,7 +68,9 @@ def read_build_id(path: Path) -> str | None: return None handle.seek(0) return _build_id_from(ELFFile(handle)) - except (OSError, Exception): # noqa: B014 - a malformed file is not ours to fix + except (OSError, ELFError): + # Scanning a machine's libraries meets files that only look like ELF. + # Anything else raising here is a bug in this parser, not bad input. return None diff --git a/scripts/sbom/sbom/evidence.py b/scripts/sbom/sbom/evidence.py index 7a1eb98284..afea193dfb 100644 --- a/scripts/sbom/sbom/evidence.py +++ b/scripts/sbom/sbom/evidence.py @@ -171,12 +171,34 @@ def discover(repo_root: Path, build_dir: Path) -> Discovery: f"{deps_dir} not found; configure the build before collecting evidence" ) - config = cache.get("CMAKE_BUILD_TYPE") or "Release" + # Guessing a configuration would point the build graph and the staged-tree + # probe at a directory that need not exist, and every member they would have + # explained then reads as having reached the wheel by an unknown route. + config = cache.get("CMAKE_BUILD_TYPE") + if not config: + raise EvidenceError( + f"{build_dir}/CMakeCache.txt states no CMAKE_BUILD_TYPE; the build " + "graph and the staged tree are both read per configuration" + ) graph = BuildGraph(build_dir, config) - components = discover_source_trees(deps_dir) - archives = discover_archives( - repo_root, skip=(build_dir, repo_root / "build-wheel", repo_root / "dist") + # Everything that is a build tree rather than source, skipped the same way + # everywhere it matters. The active one is given; any other is found by the + # CMakeCache.txt that makes it one. A second build directory holds staged + # copies of the repo, and walking those as source leaves a file's origin + # ambiguous rather than wrong, which surfaces as an unexplained member. + not_source = ( + build_dir, + repo_root / "build-wheel", + repo_root / "dist", + *( + path.parent + for pattern in ("*/CMakeCache.txt", "*/*/CMakeCache.txt") + for path in repo_root.glob(pattern) + ), ) + + components = discover_source_trees(deps_dir) + archives = discover_archives(repo_root, skip=not_source) components.update(_archive_components(archives, repo_root)) source_files = FileIndex() @@ -185,22 +207,7 @@ def discover(repo_root: Path, build_dir: Path) -> Discovery: source_files.add_tree(root, root.name) repo_files = FileIndex() - # Skip the build tree by path, not by name: --build-dir is free-form, and a - # differently-named one inside the repo would otherwise be walked twice. - # Any *other* configured tree is skipped too, found by the CMakeCache.txt - # that makes it one -- a second build directory holds staged copies of the - # repo, and walking them as source leaves a file's origin ambiguous rather - # than wrong, which reads as an unexplained member. - other_builds = tuple( - path.parent - for pattern in ("*/CMakeCache.txt", "*/*/CMakeCache.txt") - for path in repo_root.glob(pattern) - ) - repo_files.add_tree( - repo_root, - ".", - skip=(build_dir, *other_builds, repo_root / "build-wheel", repo_root / "dist"), - ) + repo_files.add_tree(repo_root, ".", skip=not_source) # _deps/*-src is the component-source domain, already indexed above; a wheel # member matching one is a copy of upstream, not something this build made. diff --git a/scripts/sbom/sbom/inventory.py b/scripts/sbom/sbom/inventory.py index 0ae2a4991d..63f75ceff8 100644 --- a/scripts/sbom/sbom/inventory.py +++ b/scripts/sbom/sbom/inventory.py @@ -112,7 +112,7 @@ def __init__(self, discovery: Discovery, system_resolver=None) -> None: artifact.output ) self._artifacts_by_hash.setdefault(digest, []).append(artifact) - build_id = _build_id_of(artifact.output) + build_id = elf.read_build_id(artifact.output) if build_id: self._artifacts_by_build_id.setdefault(build_id, []).append( artifact @@ -360,7 +360,7 @@ def _resolve_by_path( artifacts = [ artifact for artifact in self.graph.by_name(name) - if not _build_ids_disagree(build_id, _build_id_of(artifact.output)) + if not _build_ids_disagree(build_id, elf.read_build_id(artifact.output)) ] if len(artifacts) == 1: return self._from_artifact( @@ -460,20 +460,6 @@ def _build_ids_disagree(left: str | None, right: str | None) -> bool: return bool(left and right and left != right) -def _build_id_of(path: Path) -> str | None: - """The build-id of a file on disk, or None if it is not an ELF.""" - try: - data = path.read_bytes() - except OSError: - return None - if not elf.is_elf(data[:4]): - return None - try: - return elf.read_dynamic(data).build_id - except Exception: # noqa: BLE001 - a malformed build output is not ours to fix - return None - - def _attribute( entry: Entry, wheel: WheelInfo, diff --git a/tests/python/sbom/test_discovery.py b/tests/python/sbom/test_discovery.py index 99e6be7d41..555e053343 100644 --- a/tests/python/sbom/test_discovery.py +++ b/tests/python/sbom/test_discovery.py @@ -241,3 +241,22 @@ def test_a_fetched_archive_records_where_it_came_from(workspace, license_data): assert after.download_location.startswith("https://api.ngc.nvidia.com/") assert after.supplier == "Organization: api.ngc.nvidia.com" assert "fetched from https://" in after.source_info + + +def test_a_build_tree_with_no_configuration_says_so(workspace): + """Guessing Release would read a directory that need not exist, and every + member it would have explained becomes an unexplained one instead.""" + from sbom import evidence + + cache = workspace.build / "CMakeCache.txt" + cache.write_text( + "\n".join( + line + for line in cache.read_text().splitlines() + if not line.startswith("CMAKE_BUILD_TYPE:") + ) + + "\n" + ) + + with pytest.raises(evidence.EvidenceError, match="CMAKE_BUILD_TYPE"): + evidence.discover(workspace.root, workspace.build) diff --git a/tests/python/sbom/test_gates.py b/tests/python/sbom/test_gates.py index 3581eb3e27..c19b587c4f 100644 --- a/tests/python/sbom/test_gates.py +++ b/tests/python/sbom/test_gates.py @@ -343,3 +343,20 @@ def test_a_name_match_a_build_id_contradicts_is_refused(workspace, license_data) workspace.wheel, workspace.root / "sbom", ) + + +def test_an_unreadable_archive_says_so(workspace, license_data): + """Only archives this build fetched are indexed, so one that will not open + is a broken download -- not a stray file to walk past.""" + from sbom import discovery, licensing + + licensing.load_corpus(license_data) + (workspace.root / synth.SDK_ARCHIVE).write_bytes(b"not an archive at all") + + with pytest.raises(discovery.ArchiveError, match="cannot be read"): + build_module.build( + workspace.root, + workspace.build, + workspace.wheel, + workspace.root / "sbom", + ) From b5a7eeb3d69ea5b949618c163665ed3a8ee28cc7 Mon Sep 17 00:00:00 2001 From: Andrei Aristarkhov Date: Wed, 30 Sep 2026 19:49:41 -0700 Subject: [PATCH 012/101] build(sbom): stop treating identical bytes as proof of origin A stock Apache-2.0 text is the same file in every project that ships one, and a module that is nothing but an SPDX header is the same in every project using that header. Resolving by content hash without a uniqueness check named whichever candidate was indexed first, as a flat assertion. Three claims were wrong: MUJOCO_LICENSE was a copy of flatbuffers, a first-party module was a copy of a dependency, and seven CloudXR files came from the other SDK. The first leaked into THIRD-PARTY-NOTICES.md, filing mujoco's redistribution obligation under flatbuffers. Content may now name a component only when the digest resolves to exactly one -- this repository counting as a candidate, which is what distinguishes a first-party file. Otherwise the path decides, and failing that the document says the bytes are shared and claims no single source. The gaps list keyed on whether anything matched, so a component whose match could not be expressed carried a LicenseRef without being reported. It keys on the conclusion now, and the warning states which of the two reasons applies. A VERSION member with two lines put a newline inside versionInfo, which breaks every tag-value serialisation and table rendering downstream. Signed-off-by: Andrei Aristarkhov --- scripts/sbom/sbom/cli.py | 3 +-- scripts/sbom/sbom/discovery.py | 14 +++++++--- scripts/sbom/sbom/inventory.py | 46 +++++++++++++++++++++++++++++++++ scripts/sbom/sbom/report.py | 18 ++++++++++--- tests/python/sbom/test_gates.py | 36 ++++++++++++++++++++++++++ 5 files changed, 109 insertions(+), 8 deletions(-) diff --git a/scripts/sbom/sbom/cli.py b/scripts/sbom/sbom/cli.py index bee76a5705..3718d544c8 100644 --- a/scripts/sbom/sbom/cli.py +++ b/scripts/sbom/sbom/cli.py @@ -83,8 +83,7 @@ def _cmd_build(args: argparse.Namespace) -> int: for item in merged["wheels"]: for entry in item["components_with_unidentified_license"]: print( - f"::warning::{item['filename']}: {entry['component']} ships a license " - "text that matches no SPDX reference" + f"::warning::{item['filename']}: {entry['component']}: {entry['reason']}" ) return 0 diff --git a/scripts/sbom/sbom/discovery.py b/scripts/sbom/sbom/discovery.py index fd7b3821af..67033f19a2 100644 --- a/scripts/sbom/sbom/discovery.py +++ b/scripts/sbom/sbom/discovery.py @@ -781,6 +781,7 @@ class ArchiveIndex: def __init__(self) -> None: self.by_hash: dict[str, Member] = {} + self.all_by_hash: dict[str, list[Member]] = {} self.by_name: dict[str, list[Member]] = {} # Survives the RPATH/SONAME rewrite a repair tool applies on the way in, # so a patched wheel member still points at the archive it came from. @@ -807,6 +808,7 @@ def add_archive(self, path: Path, display: str) -> None: member = replace(member, build_id=build_id) self.by_build_id.setdefault(build_id, member) self.by_hash.setdefault(member.sha256, member) + self.all_by_hash.setdefault(member.sha256, []).append(member) self.by_name.setdefault(Path(member.path).name, []).append(member) self._maybe_license(display, member, data) @@ -815,9 +817,10 @@ def _maybe_license(self, display: str, member: Member, data: bytes) -> None: if Path(member.path).parent.as_posix() not in {".", ""}: return # only the archive root states the bundle's own terms if name.upper() == "VERSION" and len(data) < 256: - self.archives[display]["version_text"] = data.decode( - "utf-8", "replace" - ).strip() + # First line only. These files carry more than one, and a newline in + # a version breaks every tag-value and table rendering downstream. + text = data.decode("utf-8", "replace").strip().splitlines() + self.archives[display]["version_text"] = text[0].strip() if text else "" return if not licensing._classify(name): # noqa: SLF001 - same package return @@ -880,6 +883,10 @@ class FileIndex: def __init__(self) -> None: self.by_hash: dict[str, str] = {} + # Every path sharing a digest, because one is not evidence of origin when + # several files hold the same bytes -- a stock licence text, or a module + # that is nothing but an SPDX header. + self.all_by_hash: dict[str, list[str]] = {} self.by_suffix: dict[str, list[str]] = {} self._digests: dict[str, str] = {} @@ -902,6 +909,7 @@ def add_tree( continue self._digests[str(path.resolve())] = digest self.by_hash.setdefault(digest, display) + self.all_by_hash.setdefault(digest, []).append(display) # Two trailing segments is enough to disambiguate an __init__.py. self.by_suffix.setdefault( "/".join(relative.rsplit("/", 2)[-2:]), [] diff --git a/scripts/sbom/sbom/inventory.py b/scripts/sbom/sbom/inventory.py index 63f75ceff8..3509750da2 100644 --- a/scripts/sbom/sbom/inventory.py +++ b/scripts/sbom/sbom/inventory.py @@ -167,6 +167,17 @@ def resolve( artifacts[0], wheel_path, f"{source}; exact build output" ) + # Identical bytes are not evidence of origin when several components + # hold them: a stock Apache-2.0 text is the same file in every project + # that ships one. Say so, and let the path decide, rather than naming + # whichever happened to be indexed first. + shared = self._shared_content(candidate_digest) + if shared is not None: + by_path = self._resolve_by_path(wheel_path, source, build_id) + if by_path is not None: + return by_path + return shared(wheel_path, source) + component_file = self.source_files.by_hash.get(candidate_digest) if component_file: key = component_file.split("/", 1)[0].removesuffix("-src") @@ -274,6 +285,41 @@ def _from_build_host(self, wheel_path: str, soname: str) -> Attribution | None: primary=component.key, ) + def _shared_content(self, digest: str): + """How to describe content that more than one component also holds. + + Returns None when the digest names exactly one component, which is the + case worth stating as a fact. Otherwise it returns a builder that names + every candidate and claims none of them, keeping each component's + licence obligation without asserting a copy that did not happen. + """ + owners: dict[str, str] = {} + for display in self.source_files.all_by_hash.get(digest, []): + owners[display.split("/", 1)[0].removesuffix("-src")] = display + for member in self.archives.all_by_hash.get(digest, []): + owners[member.container] = f"{member.path} in {member.container}" + # This repository holding the same bytes counts as a candidate too: a + # module that is nothing but an SPDX header is identical in every project + # that uses the same header, and calling ours a copy of a dependency's + # would ship a first-party file as redistributed third-party content. + ours = self.repo_files.all_by_hash.get(digest, []) + if len(owners) + bool(ours) < 2: + return None + + def build(wheel_path: str, source: str) -> Attribution: + where = ", ".join(sorted([*owners.values(), *ours])) + return Attribution( + path=wheel_path, + origin="shared-content", + detail=( + f"{source}; these bytes are held by more than one component " + f"({where}); the build does not record which supplied this file" + ), + components={key: {"shared-content"} for key in owners}, + ) + + return build + def _resolve_by_build_id( self, wheel_path: str, source: str, build_id: str ) -> Attribution | None: diff --git a/scripts/sbom/sbom/report.py b/scripts/sbom/sbom/report.py index 5adff7dbe6..85de8ecf2c 100644 --- a/scripts/sbom/sbom/report.py +++ b/scripts/sbom/sbom/report.py @@ -17,6 +17,9 @@ "derived-file": "files were copied in and modified by the build", "extracted-file": "files were unpacked from its archive", "vendored-library": "the library was vendored in by auditwheel", + "shared-content": ( + "a shipped file holds bytes this component also holds, and so do others" + ), } _SAFE_NAME = re.compile(r"[^A-Za-z0-9._-]") @@ -101,13 +104,22 @@ def report(inventory: Inventory, evidence_doc: dict, wheel_name: str) -> dict: "reason": "no license grant found in the component's own files", } ) - elif not any(item.identified for item in grants): + elif "LicenseRef-" in component.license_concluded: + # Keyed on the conclusion, not on whether anything matched: a text + # can match and still be unusable, when the id it matched is newer + # than the expression grammar a consumer validates with. Either way + # the terms are only readable as the text this wheel carries. unidentified.append( { "component": key, "reason": ( - "the shipped grant matches no text in the SPDX reference " - "corpus; it is reproduced verbatim in the document" + "the shipped grant is carried verbatim rather than named: " + + ( + "it matches no text in the SPDX reference corpus" + if not any(item.identified for item in grants) + else "what it matches cannot be expressed in the SPDX " + "grammar shipped with this document" + ) ), "evidence": [item.path for item in grants], } diff --git a/tests/python/sbom/test_gates.py b/tests/python/sbom/test_gates.py index c19b587c4f..fed90279fa 100644 --- a/tests/python/sbom/test_gates.py +++ b/tests/python/sbom/test_gates.py @@ -360,3 +360,39 @@ def test_an_unreadable_archive_says_so(workspace, license_data): workspace.wheel, workspace.root / "sbom", ) + + +def test_bytes_more_than_one_component_holds_name_no_single_source( + workspace, license_data +): + """A stock licence text is the same file in every project that ships one. + + Naming whichever was indexed first would file one project's redistribution + obligation under another's. Say the bytes are shared and claim no single + source instead. + """ + from sbom import licensing + + licensing.load_corpus(license_data) + shared = synth.spdx_text("Apache-2.0").encode() + for key in ("alpha", "beta"): + (workspace.build / "_deps" / f"{key}-src").mkdir(parents=True, exist_ok=True) + (workspace.build / "_deps" / f"{key}-src" / "LICENSE").write_bytes(shared) + + payload = synth.wheel_payload(workspace) + payload["isaaccapture/viz/SOMEDEP_LICENSE"] = shared + synth.write_wheel(workspace.wheel, payload) + + build_module.build( + workspace.root, workspace.build, workspace.wheel, workspace.root / "sbom" + ) + with zipfile.ZipFile(workspace.wheel) as archive: + spdx = json.loads(archive.read(f"{DIST_INFO}/sboms/{WHEEL_NAME}.spdx.json")) + entry = next( + item for item in spdx["files"] if item["fileName"].endswith("SOMEDEP_LICENSE") + ) + + assert "shared-content" in entry["comment"] + assert "more than one component" in entry["comment"] + # Both candidates named, neither claimed as the source. + assert "alpha" in entry["comment"] and "beta" in entry["comment"] From 8b9236e0790e16ab496e7520b7f17181e40dad95 Mon Sep 17 00:00:00 2001 From: Andrei Aristarkhov Date: Wed, 30 Sep 2026 19:49:42 -0700 Subject: [PATCH 013/101] licenses: restore the REUSE pool to its canonical texts `LICENSES/` is a dictionary of licence identifiers, not a statement about who owns what: REUSE expects each file to hold the licence as published, and `reuse download` fetches it with its placeholders intact. Two had been personalised. Apache-2.0's instructional appendix was filled in with this project's name, and MIT's `Copyright (c) ` line -- which is the whole of its attribution requirement -- had been deleted. Both now normalise equal to the pinned SPDX list. The grant was never modified; only the boilerplate around it. It mattered because the collector was packaging these as third-party components' licence texts, so the only notice shipped for another company's code was this project's. Signed-off-by: Andrei Aristarkhov --- LICENSES/Apache-2.0.txt | 4 ++-- LICENSES/MIT.txt | 2 ++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/LICENSES/Apache-2.0.txt b/LICENSES/Apache-2.0.txt index 1a89b9054d..261eeb9e9f 100644 --- a/LICENSES/Apache-2.0.txt +++ b/LICENSES/Apache-2.0.txt @@ -178,7 +178,7 @@ APPENDIX: How to apply the Apache License to your work. To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "{}" + boilerplate notice, with the fields enclosed by brackets "[]" replaced with your own identifying information. (Don't include the brackets!) The text should be enclosed in the appropriate comment syntax for the file format. We also recommend that a @@ -186,7 +186,7 @@ same "printed page" as the copyright notice for easier identification within third-party archives. - Copyright 2020 NVIDIA Corporation + Copyright [yyyy] [name of copyright owner] Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/LICENSES/MIT.txt b/LICENSES/MIT.txt index 9cf106272a..242da6233b 100644 --- a/LICENSES/MIT.txt +++ b/LICENSES/MIT.txt @@ -1,5 +1,7 @@ MIT License +Copyright (c) + Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights From fdd770b35a3eda0d7d4b70ef3c8a3065fb4be43c Mon Sep 17 00:00:00 2001 From: Andrei Aristarkhov Date: Wed, 30 Sep 2026 19:49:43 -0700 Subject: [PATCH 014/101] build(sbom): package a reference text, never one from this checkout A vendored component states an identifier in its headers and ships no licence file, so something has to supply the text. It took this repository's REUSE pool, because a file named after the identifier was already on disk there -- and the pinned SPDX corpus, which is the authoritative copy, was discarded at load time in favour of the n-grams used for matching. The corpus keeps its texts now and supplies them. The pool licenses this project and carries its notices; packaging one as a third party's terms put this project's copyright above their code. That leaves the reference text's placeholder where a holder belongs, so the copyright line the headers state is recorded rather than dropped -- it is already parsed, since naming the component is what it is parsed for. Declaring no copyright beside a sourceInfo reading "states its own copyright" was the same document contradicting itself. The reference says why these rules exist rather than only what they are, and covers the component that has no licence file of its own. Signed-off-by: Andrei Aristarkhov --- docs/source/references/sbom.rst | 22 ++++++++++++++----- scripts/sbom/sbom/discovery.py | 22 ++++++++++++++++++- scripts/sbom/sbom/document.py | 2 +- scripts/sbom/sbom/licensing.py | 34 +++++++++++++++++++---------- tests/python/sbom/test_discovery.py | 10 ++++++--- 5 files changed, 67 insertions(+), 23 deletions(-) diff --git a/docs/source/references/sbom.rst b/docs/source/references/sbom.rst index db3ee791ec..80c8f197b8 100644 --- a/docs/source/references/sbom.rst +++ b/docs/source/references/sbom.rst @@ -184,12 +184,22 @@ shipping nothing. Pinning is by commit, so no tag or branch upstream can move un build, and each report records the commit its identifications were made against. Consulting the list is not shipping it. Every current identifier is available to match -against, so a dependency arriving under any of them lands with a real SPDX id. What the -wheel carries for a third-party component is that component's own license file, verbatim, -copyright holders and all; a reference text is never substituted for it and no file from -the checkout is packaged. The canonical texts that do ship under -``.dist-info/licenses/LICENSES/`` are this repository's own REUSE pool — the texts the -SPDX headers on its sources name — and have nothing to do with the matching. +against, so a dependency arriving under any of them lands with a real SPDX id. + +What the wheel carries for a third-party component is that component's own license file, +verbatim, copyright holders and all. Substituting a reference text would convey the terms +and drop the notice — for MIT and the BSD licenses those are the same file, differing by +the one line that names who licensed it to you — and it would replace the component's +actual grant with our reading of it, which is what the matching is for and not what it +decides. Where a component has *no* license file, and only states an identifier in its +source headers, there is nothing of its own to package: the text that identifier names in +the pinned list ships instead, together with the copyright line those headers state. + +Never a file from this checkout. ``LICENSES/`` holds this repository's REUSE pool, whose +copies exist to license *this* project and carry its notices; packaging one as a third +party's terms would put our copyright above their code. Those texts ship under +``.dist-info/licenses/LICENSES/`` as the project's own, and have nothing to do with the +matching or with any other component. A match needs a canonical license text to be almost entirely present *and* to account for a real share of the file, which is what stops a license quoted in the appendix of a long diff --git a/scripts/sbom/sbom/discovery.py b/scripts/sbom/sbom/discovery.py index 67033f19a2..284281c03c 100644 --- a/scripts/sbom/sbom/discovery.py +++ b/scripts/sbom/sbom/discovery.py @@ -111,6 +111,9 @@ class Component: root: Path | None = None license_concluded: str = "NOASSERTION" license_declared: str = "NOASSERTION" + # The component's own copyright notice, where the build can read one. Naming + # a holder and then declaring none would contradict the same document. + copyright_text: str = "NOASSERTION" evidence: tuple[licensing.LicenseEvidence, ...] = () @@ -497,6 +500,20 @@ def _holder_of(path: Path) -> tuple[str, str | None] | None: return (holder, licensing.read_spdx_tag(head)) if holder else None +def _notice_of(path: Path) -> str | None: + """The copyright line a file states, as written.""" + try: + head = path.open("rb").read(4096).decode("utf-8", "replace") + except OSError: + return None + match = _COPYRIGHT_TAG.search(head) + if not match: + return None + notice = " ".join(match.group(0).split()).lstrip("/ *#") + # The tag introduces the notice; it is not part of it. + return re.sub(r"^SPDX-FileCopyrightText:\s*", "", notice) + + def discover_vendored( repo_root: Path, candidates: set[Path], authors: set[str] ) -> tuple[dict[str, Component], dict[str, list[Path]]]: @@ -504,6 +521,7 @@ def discover_vendored( grouped: dict[str, list[Path]] = {} licences: dict[str, set[str]] = {} names: dict[str, str] = {} + notices: dict[str, set[str]] = {} for path in sorted(candidates): stated = _holder_of(path) @@ -515,12 +533,13 @@ def discover_vendored( key = f"vendored:{licensing.spdx_safe(holder).lower()}" grouped.setdefault(key, []).append(path) names[key] = holder + notices.setdefault(key, set()).add(_notice_of(path) or holder) if expression: licences.setdefault(key, set()).add(expression) components: dict[str, Component] = {} for key, paths in grouped.items(): - evidence = licensing.pool_evidence(repo_root, key, licences.get(key, set())) + evidence = licensing.canonical_evidence(key, licences.get(key, set())) concluded, declared = licensing.expression(evidence, key) if licences.get(key): declared = licensing.combine(sorted(licences[key])) @@ -544,6 +563,7 @@ def discover_vendored( ), license_concluded=concluded, license_declared=declared, + copyright_text="\n".join(sorted(notices.get(key, {names[key]}))), evidence=tuple(evidence), ) return components, grouped diff --git a/scripts/sbom/sbom/document.py b/scripts/sbom/sbom/document.py index 7d917bf9f4..f9a87598ac 100644 --- a/scripts/sbom/sbom/document.py +++ b/scripts/sbom/sbom/document.py @@ -65,7 +65,7 @@ def _component_package(component: Component, roles: set[str]) -> dict: "filesAnalyzed": False, "licenseConcluded": component.license_concluded, "licenseDeclared": component.license_declared, - "copyrightText": NOASSERTION, + "copyrightText": component.copyright_text, "externalRefs": [ { "referenceCategory": "PACKAGE-MANAGER", diff --git a/scripts/sbom/sbom/licensing.py b/scripts/sbom/sbom/licensing.py index dad49ffbd3..5421667c99 100644 --- a/scripts/sbom/sbom/licensing.py +++ b/scripts/sbom/sbom/licensing.py @@ -126,10 +126,16 @@ def _ngrams(words: list[str]) -> set[str]: @dataclass(frozen=True) class Reference: - """One SPDX licence, normalized for comparison.""" + """One SPDX licence, normalized for comparison and kept verbatim. + + The text is retained because it is the only authoritative copy of a licence + this build has: a component that states an identifier and ships no file of + its own has to be packaged with the text that identifier names. + """ name: str grams: frozenset[str] + text: str = "" @property def size(self) -> int: @@ -185,6 +191,7 @@ def load_corpus(json_dir: Path) -> str: entries[payload["licenseId"]] = Reference( name=payload.get("name", payload["licenseId"]), grams=frozenset(grams), + text=payload.get("licenseText", ""), ) if not entries: @@ -474,31 +481,34 @@ def combine(expressions: list[str]) -> str: return _combine(expressions) -def pool_evidence( - repo_root: Path, component_key: str, expressions: set[str] +def canonical_evidence( + component_key: str, expressions: set[str] ) -> list[LicenseEvidence]: - """Texts from the repository's REUSE pool for the ids a file declares. + """The SPDX reference text for the ids a vendored file declares about itself. - A vendored file states an identifier and nothing else; LICENSES/ is where - this repository already keeps the matching text, which is what the pool is - for. + Such a component ships no licence file, so there is nothing of its own to + package verbatim and the text its identifier names is the only honest + substitute. It comes from the list this build pinned and fetched -- never + from this repository's own REUSE pool, whose copies exist to license *this* + project and carry its notices, not the component's. """ found: list[LicenseEvidence] = [] seen: set[str] = set() + entries = corpus()[1] for expression in sorted(expressions): for token in _spdx_licensing().license_keys( _spdx_licensing().parse(expression, validate=False) ): - candidate = repo_root / "LICENSES" / f"{token}.txt" - if token in seen or not candidate.is_file(): + reference = entries.get(token) + if token in seen or reference is None or not reference.text: continue seen.add(token) found.append( _evidence( component_key, - "reuse-pool", - f"LICENSES/{token}.txt", - candidate.read_text(encoding="utf-8", errors="replace"), + "spdx-reference", + f"{token}.txt", + reference.text, "grant", ) ) diff --git a/tests/python/sbom/test_discovery.py b/tests/python/sbom/test_discovery.py index 555e053343..5fc47fff66 100644 --- a/tests/python/sbom/test_discovery.py +++ b/tests/python/sbom/test_discovery.py @@ -151,9 +151,13 @@ def test_third_party_checked_in_here_is_still_third_party(workspace): assert [c.name for c in vendored.values()] == ["Upstream Widgets Ltd"] component = next(iter(vendored.values())) assert component.license_concluded == "BSL-1.0" - # Its text comes from this repository's REUSE pool, which is what the pool - # is for: the file states an identifier and carries no text of its own. - assert [item.path for item in component.evidence] == ["LICENSES/BSL-1.0.txt"] + # The file states an identifier and carries no text of its own, so the text + # comes from the SPDX list this build pinned -- never from this repository's + # own REUSE pool, whose copies license this project and carry its notices. + assert [item.path for item in component.evidence] == ["BSL-1.0.txt"] + assert all(item.origin == "spdx-reference" for item in component.evidence) + # The notice the component states about itself is recorded, not dropped. + assert "Upstream Widgets Ltd" in component.copyright_text def test_a_vendored_header_is_credited_to_the_artifact_that_includes_it(workspace): From 3054b8a7eb6d9e642b1cf6bcf81440c715186489 Mon Sep 17 00:00:00 2001 From: Andrei Aristarkhov Date: Wed, 30 Sep 2026 19:50:31 -0700 Subject: [PATCH 015/101] build: ignore coverage databases `python -m coverage` writes `.coverage` into whichever directory the suite ran in, so measuring a test leaf leaves one behind under `tests/`. Nothing ignored it and one was committed. Signed-off-by: Andrei Aristarkhov --- .gitignore | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.gitignore b/.gitignore index ae5e75dbdf..c7ce6a078e 100644 --- a/.gitignore +++ b/.gitignore @@ -70,6 +70,10 @@ MUJOCO_LOG.TXT # Runtime device files injected by MCP server infrastructure .mcp.json +# Coverage databases, written beside whichever suite was measured +.coverage +.coverage.* + # Agent session logs (never committed) tmp/ From d36bda874980ea87d3b78bc9e7734c7842e2ba74 Mon Sep 17 00:00:00 2001 From: Andrei Aristarkhov Date: Wed, 30 Sep 2026 20:12:34 -0700 Subject: [PATCH 016/101] build(sbom): tell repository source from this build's own output A second audit of a real wheel found the origin layer citing things that are not sources. Repository files were indexed by walking the working tree, skipping only directories holding a CMakeCache.txt. An install prefix carries no such marker, so two of them were walked as source and offered as candidate origins -- one belonging to a different build than the document describes. Git knows which files this repository holds, so that is what is indexed now. A dependency's commit came from `git -C rev-parse HEAD`, which answers for whichever repository encloses the directory. A checkout carrying no .git of its own was therefore stamped with this project's commit. The directory has to be the root of the checkout it is credited to. Build-tree citations were written as `build/...` whatever the directory was called, so following one resolved to the wrong tree or to nothing. They name the directory that was read. Bytes several components hold are not evidence the wheel contains any of their files: `shared-content` no longer earns the component a CONTAINS edge, which the wheel was claiming for flatbuffers on the strength of one stock Apache-2.0 text. Such a file has no single source and so no primary, which kept it out of the relationship graph entirely -- it is now tied to each candidate as OTHER, rather than stating the ambiguity only in prose no tool reads. Signed-off-by: Andrei Aristarkhov --- scripts/sbom/sbom/discovery.py | 51 +++++++++++++++++++++++++++++++-- scripts/sbom/sbom/document.py | 18 +++++++++++- scripts/sbom/sbom/evidence.py | 11 +++++-- scripts/sbom/sbom/inventory.py | 8 ++++++ scripts/sbom/sbom/licensing.py | 5 ++-- tests/python/sbom/synth.py | 33 +++++++++++++++++++++ tests/python/sbom/test_gates.py | 1 + 7 files changed, 119 insertions(+), 8 deletions(-) diff --git a/scripts/sbom/sbom/discovery.py b/scripts/sbom/sbom/discovery.py index 284281c03c..67d73177f7 100644 --- a/scripts/sbom/sbom/discovery.py +++ b/scripts/sbom/sbom/discovery.py @@ -117,6 +117,19 @@ class Component: evidence: tuple[licensing.LicenseEvidence, ...] = () +def repo_head(directory: Path) -> str | None: + """The commit of the checkout rooted at `directory`, if it is one. + + `git -C rev-parse HEAD` answers for whichever repository encloses the + directory, so a dependency whose checkout carries no .git of its own would + otherwise be stamped with this project's commit. + """ + top = git(directory, "rev-parse", "--show-toplevel") + if top is None or Path(top).resolve() != directory.resolve(): + return None + return git(directory, "rev-parse", "HEAD") + + def git(repo: Path, *args: str) -> str | None: """Run a read-only git query, or return None if git cannot answer.""" try: @@ -193,7 +206,7 @@ def discover_source_trees(deps_dir: Path) -> dict[str, Component]: continue key = root.name.removesuffix("-src") declared = _declared_constraint(deps_dir, key) - commit = git(root, "rev-parse", "HEAD") + commit = repo_head(root) remote = git(root, "config", "--get", "remote.origin.url") or declared["url"] describe = git(root, "describe", "--tags", "--always") @@ -884,6 +897,23 @@ def take(name: str, data: bytes) -> None: return collected +def tracked_files(repo_root: Path) -> list[str]: + """What this repository actually holds, as git records it. + + A working tree also holds whatever was built in it. An install prefix has no + marker saying so, so walking the directory cannot tell the two apart, and a + wheel member matching a *build output* would be reported as a copy of + repository source. Git knows the difference. + """ + listing = git(repo_root, "ls-files", "-z", "--cached") + if listing is None: + raise FileApiError( + f"{repo_root} is not a git checkout this build can query; the SBOM " + "distinguishes repository source from build output by what git tracks" + ) + return [item for item in listing.split("\0") if item] + + def discover_archives(repo_root: Path, skip: tuple[Path, ...] = ()) -> ArchiveIndex: """Index archives held in the repository, not ones a build unpacked.""" index = ArchiveIndex() @@ -918,8 +948,23 @@ def add_tree( self, root: Path, display_root: str, skip: tuple[Path, ...] = () ) -> None: skipped = tuple(item.resolve() for item in skip) - for path in _walk(root): - if any(str(path).startswith(str(item)) for item in skipped): + self._add( + root, + display_root, + ( + path + for path in _walk(root) + if not any(str(path).startswith(str(item)) for item in skipped) + ), + ) + + def add_paths(self, root: Path, display_root: str, paths) -> None: + """Index an explicit list rather than whatever the tree happens to hold.""" + self._add(root, display_root, (root / item for item in paths)) + + def _add(self, root: Path, display_root: str, paths) -> None: + for path in paths: + if not path.is_file() or path.is_symlink(): continue relative = path.relative_to(root).as_posix() display = f"{display_root}/{relative}" diff --git a/scripts/sbom/sbom/document.py b/scripts/sbom/sbom/document.py index f9a87598ac..33a6bb9280 100644 --- a/scripts/sbom/sbom/document.py +++ b/scripts/sbom/sbom/document.py @@ -21,6 +21,9 @@ NOASSERTION = "NOASSERTION" # How a component reaches the wheel decides the relationship that says so. _LINK_HOWS = {"compiled-source", "header-include", "prebuilt-library"} +# Bytes this component also holds, which is not evidence the wheel carries its +# file. It states a candidate, so it earns no CONTAINS and no COPY_OF. +_AMBIGUOUS_HOWS = {"shared-content"} def _spdx_id(prefix: str, value: str) -> str: @@ -262,7 +265,7 @@ def build_document( "relationshipType": "STATIC_LINK", } ) - if roles - _LINK_HOWS: + if roles - _LINK_HOWS - _AMBIGUOUS_HOWS: relationships.append( { "spdxElementId": wheel_id, @@ -284,6 +287,19 @@ def build_document( } ) + # A file whose bytes several components hold names no single source, so + # it has no primary and `files_of` does not see it. Stating the candidates + # only in prose would leave it attached to nothing any tool reads. + for name in inventory.files_sharing(key): + if name in file_ids: + relationships.append( + { + "spdxElementId": file_ids[name], + "relatedSpdxElement": package_id, + "relationshipType": "OTHER", + } + ) + for soname, consumers in inventory.external_runtime.items(): package_id = _spdx_id("Package-external", soname) packages.append( diff --git a/scripts/sbom/sbom/evidence.py b/scripts/sbom/sbom/evidence.py index afea193dfb..d67b9369a4 100644 --- a/scripts/sbom/sbom/evidence.py +++ b/scripts/sbom/sbom/evidence.py @@ -27,6 +27,7 @@ discover_vendored, project_authors, discover_source_trees, + tracked_files, EMPTY_SHA256, _walk, ) @@ -206,8 +207,11 @@ def discover(repo_root: Path, build_dir: Path) -> Discovery: if root.is_dir(): source_files.add_tree(root, root.name) + # Tracked files only: an install prefix carries no marker distinguishing it + # from source, so a directory walk cannot, and a wheel member matching this + # build's own output would be reported as a copy of repository source. repo_files = FileIndex() - repo_files.add_tree(repo_root, ".", skip=not_source) + repo_files.add_paths(repo_root, ".", tracked_files(repo_root)) # _deps/*-src is the component-source domain, already indexed above; a wheel # member matching one is a copy of upstream, not something this build made. @@ -215,10 +219,13 @@ def discover(repo_root: Path, build_dir: Path) -> Discovery: # build trees, and trees a dependency stages for packaging -- so it belongs # in build_files like any other generated file. The staged tree stays out: # indexing it would let the wheel explain itself. + # Named after the directory that was read: `--build-dir` is free-form, and a + # citation reading `build/...` resolves to the wrong tree, or to nothing, for + # anyone whose build directory is not called that. build_files = FileIndex() build_files.add_tree( build_dir, - "build", + build_dir.name, skip=(*sorted(deps_dir.glob("*-src")), build_dir / "python_package"), ) diff --git a/scripts/sbom/sbom/inventory.py b/scripts/sbom/sbom/inventory.py index 3509750da2..81adb890e2 100644 --- a/scripts/sbom/sbom/inventory.py +++ b/scripts/sbom/sbom/inventory.py @@ -86,6 +86,14 @@ def files_of(self, key: str) -> list[str]: path for path, item in self.attributions.items() if item.primary == key ) + def files_sharing(self, key: str) -> list[str]: + """Files naming this component as one candidate among several.""" + return sorted( + path + for path, item in self.attributions.items() + if item.primary is None and "shared-content" in item.components.get(key, ()) + ) + class Resolver: """Probes a candidate's bytes and path against every discovered index.""" diff --git a/scripts/sbom/sbom/licensing.py b/scripts/sbom/sbom/licensing.py index 5421667c99..67e0d5308c 100644 --- a/scripts/sbom/sbom/licensing.py +++ b/scripts/sbom/sbom/licensing.py @@ -217,9 +217,10 @@ def load_corpus(json_dir: Path) -> str: def _checkout_commit(source_dir: Path) -> str: """The commit deps/third_party pinned, read back from the checkout itself.""" - from .discovery import git - return git(source_dir, "rev-parse", "HEAD") or "NOASSERTION" + from .discovery import repo_head + + return repo_head(source_dir) or "NOASSERTION" def corpus() -> tuple[str, dict[str, Reference]]: diff --git a/tests/python/sbom/synth.py b/tests/python/sbom/synth.py index 783724c30b..0c8f57265d 100644 --- a/tests/python/sbom/synth.py +++ b/tests/python/sbom/synth.py @@ -22,8 +22,10 @@ import gzip import hashlib import io +import os import json import struct +import subprocess import tarfile import zipfile from pathlib import Path @@ -271,6 +273,36 @@ def __init__(self, root: Path) -> None: self.wheel = root / "dist" / WHEEL_NAME +def _git_checkout(root: Path, initialise: bool = True) -> None: + """Make the generated project a real checkout. + + The collector separates repository source from build output by what git + tracks, because an install prefix carries no marker saying it is output. + `build/` and `dist/` are ignored here for the same reason they are in the + real repository. + """ + _write(root / ".gitignore", "build/\ndist/\n") + env = { + **os.environ, + "GIT_AUTHOR_NAME": "synth", + "GIT_AUTHOR_EMAIL": "synth@example.invalid", + "GIT_COMMITTER_NAME": "synth", + "GIT_COMMITTER_EMAIL": "synth@example.invalid", + } + run = lambda *args: subprocess.run( # noqa: E731 - local shorthand + ["git", "-C", str(root), *args], check=True, capture_output=True, env=env + ) + if initialise: + run("init", "-q", "-b", "main") + run("add", "-A") + run("commit", "-q", "--allow-empty", "-m", "synthetic project") + + +def track(root: Path) -> None: + """Commit whatever a test added after the project was generated.""" + _git_checkout(root, initialise=False) + + def _write(path: Path, data: bytes | str) -> None: path.parent.mkdir(parents=True, exist_ok=True) if isinstance(data, str): @@ -394,6 +426,7 @@ def create(root: Path) -> Workspace: _file_api_reply( root, build, (workspace.staged / extension).relative_to(build).as_posix() ) + _git_checkout(root) _write(build / "tests/unit_tests", b"\x7fELF-not-really") return workspace diff --git a/tests/python/sbom/test_gates.py b/tests/python/sbom/test_gates.py index fed90279fa..85dd0fa8c5 100644 --- a/tests/python/sbom/test_gates.py +++ b/tests/python/sbom/test_gates.py @@ -291,6 +291,7 @@ def test_a_malformed_reuse_tag_does_not_reach_the_document(workspace): # Authored sources, staged into the package like any other .py. (workspace.root / "src/python/isaaccapture" / name).write_text(text) (workspace.staged / "isaaccapture" / name).write_text(text) + synth.track(workspace.root) _rebuild(workspace, payload) with zipfile.ZipFile(workspace.wheel) as archive: From db54aea07306c4d41763b16de2c39a0c31f8bd40 Mon Sep 17 00:00:00 2001 From: Andrei Aristarkhov Date: Wed, 30 Sep 2026 20:18:54 -0700 Subject: [PATCH 017/101] build(sbom): stop asserting licence facts the evidence does not carry Both audits of the regenerated wheel landed on the same class of defect: fields stating more, or less, than what was read. `licenseDeclared` echoed the conclusion for every component without a REUSE tag, asserting a declaration nobody made and erasing the one thing the field exists for -- showing where supplier and collector disagree. Where a tag did exist it was promoted whatever file carried it, so OpenXR-SDK, a statically linked Apache-2.0 SDK, was declared CC-BY-4.0 on the strength of the tag licensing its COPYING.adoc prose. A tag speaks for the package only where the file carrying it is a licence in its own right; otherwise the field is unasserted. `licenseInfoFromFiles` said NOASSERTION while 140 files in the same document recorded Apache-2.0, which is the roll-up contradicting what it rolls up. Every file said `copyrightText: NOASSERTION` although 140 state a notice on the line above the identifier the collector already parses. It is read now rather than discarded. A vendored system library reported no version and a `pkg:generic` purl that resolves to nothing, while dpkg -- already queried for the package name -- knows `0.12.1-1build1.1`. Both now carry a `pkg:deb` purl a consumer can act on. Signed-off-by: Andrei Aristarkhov --- scripts/sbom/sbom/discovery.py | 43 ++++++++++++++++++++++++++++++++-- scripts/sbom/sbom/document.py | 10 ++++++-- scripts/sbom/sbom/inventory.py | 9 +++++++ scripts/sbom/sbom/licensing.py | 19 +++++++++++++-- 4 files changed, 75 insertions(+), 6 deletions(-) diff --git a/scripts/sbom/sbom/discovery.py b/scripts/sbom/sbom/discovery.py index 67d73177f7..fe63e898d3 100644 --- a/scripts/sbom/sbom/discovery.py +++ b/scripts/sbom/sbom/discovery.py @@ -704,12 +704,41 @@ def _identify_host_package(library: Path, origin: dict) -> dict: package = result.stdout.split(":", 1)[0].strip() origin["package"] = package origin["supplier"] = _package_supplier(package) + origin["version"] = _package_field(package, "Version") + origin["distro"] = _os_release_id() copyright_file = Path("/usr/share/doc") / package / "copyright" if copyright_file.is_file(): origin["copyright"] = str(copyright_file) return origin +def _package_field(package: str, field: str) -> str | None: + """One field of a distro package's metadata, as dpkg records it.""" + try: + result = subprocess.run( + ["dpkg-query", "-W", f"-f=${{{field}}}", package], + capture_output=True, + text=True, + check=False, + timeout=60, + ) + except (OSError, subprocess.SubprocessError): + return None + value = result.stdout.strip() + return value if result.returncode == 0 and value else None + + +def _os_release_id() -> str | None: + """The distribution this package manager belongs to, for the purl type.""" + try: + for line in Path("/etc/os-release").read_text().splitlines(): + if line.startswith("ID="): + return line.partition("=")[2].strip().strip('"') or None + except OSError: + return None + return None + + def _package_supplier(package: str) -> str | None: """Who ships a distro package, in the form SPDX wants. @@ -783,8 +812,18 @@ def system_component(soname: str, origin: dict) -> Component: kind="system-library", supplier=origin.get("supplier") or "NOASSERTION", homepage="NOASSERTION", - purl=PackageURL(type="generic", name=soname).to_string(), - version="NOASSERTION", + # A distro package the machine can name resolves; pkg:generic does not. + purl=( + PackageURL( + type="deb", + namespace=origin.get("distro"), + name=origin["package"], + version=origin.get("version"), + ).to_string() + if origin.get("package") + else PackageURL(type="generic", name=soname).to_string() + ), + version=origin.get("version") or "NOASSERTION", download_location="NOASSERTION", source_info=" ".join(details), license_concluded=concluded, diff --git a/scripts/sbom/sbom/document.py b/scripts/sbom/sbom/document.py index 33a6bb9280..b426bc6aaf 100644 --- a/scripts/sbom/sbom/document.py +++ b/scripts/sbom/sbom/document.py @@ -165,6 +165,8 @@ def build_document( file_ids: dict[str, str] = {} sha1_digests: list[str] = [] + # What the files themselves state, for the package-level roll-up below. + stated_in_files: set[str] = set() for index, entry in enumerate(sorted(wheel.entries, key=lambda item: item.name)): if entry.name in excluded_files: continue @@ -181,12 +183,16 @@ def build_document( {"algorithm": "SHA256", "checksumValue": entry.sha256}, ], "licenseConcluded": NOASSERTION, - "copyrightText": NOASSERTION, + # The notice sits directly above the identifier already parsed, so + # declaring none would discard something read and in hand. + "copyrightText": (attribution.copyright_text if attribution else None) + or NOASSERTION, } if attribution and attribution.spdx_tag: # A REUSE tag in the file is upstream stating that file's license. record["licenseConcluded"] = attribution.spdx_tag record["licenseInfoInFiles"] = [attribution.spdx_tag] + stated_in_files.add(attribution.spdx_tag) comment = [] if attribution: @@ -216,7 +222,7 @@ def build_document( "homepage": project["homepage"], "licenseConcluded": project["license"], "licenseDeclared": project["license"], - "licenseInfoFromFiles": [NOASSERTION], + "licenseInfoFromFiles": sorted(stated_in_files) or [NOASSERTION], "copyrightText": NOASSERTION, "packageVerificationCode": { "packageVerificationCodeValue": code, diff --git a/scripts/sbom/sbom/inventory.py b/scripts/sbom/sbom/inventory.py index 81adb890e2..94e0943d7f 100644 --- a/scripts/sbom/sbom/inventory.py +++ b/scripts/sbom/sbom/inventory.py @@ -57,6 +57,7 @@ class Attribution: components: dict[str, set[str]] = field(default_factory=dict) primary: str | None = None spdx_tag: str | None = None + copyright_text: str | None = None def as_json(self) -> dict: return { @@ -471,6 +472,7 @@ def build( unattributed.append(entry.name) continue attribution.spdx_tag = _spdx_tag(entry) + attribution.copyright_text = _copyright_text(entry) attributions[entry.name] = attribution roles: dict[str, set[str]] = {} @@ -533,6 +535,13 @@ def _attribute( ) +def _copyright_text(entry: Entry) -> str | None: + """The notice a file states about itself, beside the identifier.""" + if Path(entry.name).suffix.lower() not in _TAGGABLE: + return None + return licensing.read_copyright(entry.head.decode("utf-8", "replace")) + + def _spdx_tag(entry: Entry) -> str | None: """REUSE tags are upstream stating the license of that exact file.""" if Path(entry.name).suffix.lower() not in _TAGGABLE: diff --git a/scripts/sbom/sbom/licensing.py b/scripts/sbom/sbom/licensing.py index 67e0d5308c..7e6b12459b 100644 --- a/scripts/sbom/sbom/licensing.py +++ b/scripts/sbom/sbom/licensing.py @@ -50,6 +50,7 @@ _FAMILY_THRESHOLD = 0.90 # REUSE-IgnoreStart _SPDX_TAG = re.compile(r"SPDX-License-Identifier:\s*(?P[^\n\r*/#]+)") +_FILE_COPYRIGHT = re.compile(r"SPDX-FileCopyrightText:\s*(?P[^\n\r]+)") # REUSE-IgnoreEnd # Heading, then everything up to the next heading or a blank-line run. # The heading has to BE about licensing, not merely start with the word -- @@ -387,6 +388,7 @@ def _readme_fallback( return [] +NOASSERTION = "NOASSERTION" _ID_SAFE = re.compile(r"[^A-Za-z0-9.-]+") @@ -440,8 +442,15 @@ def expression( identified = [license_ref(component_key)] concluded = _combine(identified) if identified else "NOASSERTION" - tags = sorted({item.spdx_tag for item in grants if item.spdx_tag}) - declared = _combine(tags) if tags else concluded + # A tag speaks for the package only where the file carrying it is a licence + # in its own right. OpenXR's COPYING.adoc is prose about which licences the + # project uses, tagged CC-BY-4.0 for the prose and matching no licence text; + # promoting it declared a linked SDK under a documentation licence. Where no + # tag qualifies the field stays unasserted rather than echoing the + # conclusion, which would assert a declaration nobody made and hide the one + # thing the field is for -- supplier and collector disagreeing. + tags = sorted({item.spdx_tag for item in grants if item.spdx_tag and item.matches}) + declared = _combine(tags) if tags else NOASSERTION return concluded, declared @@ -477,6 +486,12 @@ def read_spdx_tag(text: str) -> str | None: return match.group("expression").strip() if match else None +def read_copyright(text: str) -> str | None: + """The notice a file states about itself, beside its REUSE tag.""" + match = _FILE_COPYRIGHT.search(text) + return " ".join(match.group("notice").split()).rstrip("*/ ") if match else None + + def combine(expressions: list[str]) -> str: """Public form of the AND-combiner, for callers assembling an expression.""" return _combine(expressions) From aa511d2f17d9cc8aecaa4089e28e0ac3d65a7db8 Mon Sep 17 00:00:00 2001 From: Andrei Aristarkhov Date: Wed, 30 Sep 2026 20:21:48 -0700 Subject: [PATCH 018/101] docs(sbom): say what the collector does, not what it was meant to do Four claims in the reference had drifted from the code, each one an audit finding rather than a wording preference. `.dist-info/sboms/` was described as holding the contents SBOM "and nothing else". It is a shared directory and auditwheel already writes a CycloneDX document of its own there, which this collector coexists with by looking for its document by name. "a dependency arriving under any of them lands with a real SPDX id" is not true when the fetched licence list runs ahead of the grammar `license-expression` ships: the id is identified but cannot be published as one, and travels as a LicenseRef. The reference now separates identifying from naming, and says which pin sets the floor. The match thresholds were described as if both were strict. The coverage bound is the looser one, and what it rules out is padding around a licence text rather than a document that is mostly one. Derived members were promised "the original digest". An archive member carries one; a file matched only by its path was never hashed. The citation now carries the digest where the input was hashed, and the reference says where that is. Signed-off-by: Andrei Aristarkhov --- docs/source/references/sbom.rst | 29 +++++++++++++++++++++-------- scripts/sbom/sbom/discovery.py | 6 ++++++ scripts/sbom/sbom/inventory.py | 6 +++++- 3 files changed, 32 insertions(+), 9 deletions(-) diff --git a/docs/source/references/sbom.rst b/docs/source/references/sbom.rst index 80c8f197b8..b7fb792dbd 100644 --- a/docs/source/references/sbom.rst +++ b/docs/source/references/sbom.rst @@ -28,9 +28,12 @@ Every text there comes from this repository or from a component the wheel redist The SPDX reference texts the collector matches against are a separate thing and are never packaged — see `How licenses are determined`_. -``.dist-info/sboms/`` holds the contents SBOM and nothing else, which is what the +``.dist-info/sboms/`` is what the `binary distribution format `_ -reserves it for. Reports and manifests are published beside the wheel instead. +reserves for SBOM documents, and it is shared: ``auditwheel`` writes a CycloneDX document +of its own there describing the libraries it vendored. This collector looks for its +document by name and leaves anything else alone. Reports and manifests are published beside +the wheel instead. Verifying a wheel ----------------- @@ -163,8 +166,9 @@ honest as the build changes. Post-processing is recorded as such. ``patchelf``, ``auditwheel``'s RPATH rewrite and the build's MJCF asset stripping all leave bytes that no longer hash to their origin, so those -members are reported as *derived from* a named input, with the original digest, rather -than as an exact copy. +members are reported as *derived from* a named input rather than as an exact copy, with +the input's digest wherever the input was hashed — which it is for an archive member, and +is not for a file matched only by its path. How licenses are determined --------------------------- @@ -184,7 +188,13 @@ shipping nothing. Pinning is by commit, so no tag or branch upstream can move un build, and each report records the commit its identifications were made against. Consulting the list is not shipping it. Every current identifier is available to match -against, so a dependency arriving under any of them lands with a real SPDX id. +against, so a dependency arriving under any of them is identified. Naming it in the +document is a second question: a document states the licence list it was built against, and +``license-expression`` — which a consumer validates with, and which +:code-file:`scripts/sbom/pyproject.toml` pins — ships its own, older copy. An id the fetched +list knows and that grammar does not cannot be published as an id, so it travels as a +``LicenseRef-`` carrying the text, and the component is listed under *License evidence gaps* +with that as the stated reason. What the wheel carries for a third-party component is that component's own license file, verbatim, copyright holders and all. Substituting a reference text would convey the terms @@ -202,9 +212,12 @@ party's terms would put our copyright above their code. Those texts ship under matching or with any other component. A match needs a canonical license text to be almost entirely present *and* to account for a -real share of the file, which is what stops a license quoted in the appendix of a long -agreement from being mistaken for that agreement's own terms. A file that genuinely carries -two licenses is reported as carrying both. +real share of the file: a tenth of a long agreement reproducing a license in an appendix is +not that agreement's own terms. The second threshold is the looser of the two, so a short +file quoting a license in full still matches — the bound it sets is on padding around a +text, not on a document that is mostly one. A file that genuinely carries two *distinct* +licenses is reported as carrying both; where one is a near-duplicate of the other, only the +match survives, and a disjunctive grant is not expressible here at all. A text that matches nothing is not guessed at. It becomes a ``LicenseRef-`` whose full text travels in the document, which says "these are the terms that shipped" without claiming to diff --git a/scripts/sbom/sbom/discovery.py b/scripts/sbom/sbom/discovery.py index fe63e898d3..4ba1e5db0a 100644 --- a/scripts/sbom/sbom/discovery.py +++ b/scripts/sbom/sbom/discovery.py @@ -978,6 +978,7 @@ def __init__(self) -> None: self.all_by_hash: dict[str, list[str]] = {} self.by_suffix: dict[str, list[str]] = {} self._digests: dict[str, str] = {} + self._by_display: dict[str, str] = {} def digest_of(self, path: Path) -> str | None: """The digest this index already computed for a path, if it indexed it.""" @@ -1014,11 +1015,16 @@ def _add(self, root: Path, display_root: str, paths) -> None: self._digests[str(path.resolve())] = digest self.by_hash.setdefault(digest, display) self.all_by_hash.setdefault(digest, []).append(display) + self._by_display[display] = digest # Two trailing segments is enough to disambiguate an __init__.py. self.by_suffix.setdefault( "/".join(relative.rsplit("/", 2)[-2:]), [] ).append(display) + def digest_of_display(self, display: str) -> str | None: + """The digest this index recorded for one of its own display paths.""" + return self._by_display.get(display) + def path_suffix_match(self, wheel_path: str) -> str | None: key = "/".join(wheel_path.rsplit("/", 2)[-2:]) matches = self.by_suffix.get(key, []) diff --git a/scripts/sbom/sbom/inventory.py b/scripts/sbom/sbom/inventory.py index 94e0943d7f..3349160c1e 100644 --- a/scripts/sbom/sbom/inventory.py +++ b/scripts/sbom/sbom/inventory.py @@ -425,10 +425,14 @@ def _resolve_by_path( component_file = self.source_files.path_suffix_match(wheel_path) if component_file: key = component_file.split("/", 1)[0].removesuffix("-src") + origin_digest = self.source_files.digest_of_display(component_file) return Attribution( path=wheel_path, origin="derived", - detail=f"{source}; derived from {component_file}", + detail=( + f"{source}; derived from {component_file}" + + (f" (origin sha256:{origin_digest})" if origin_digest else "") + ), components={key: {"derived-file"}}, primary=key, ) From 8da30ad6556a42df60aa92b63ed283f369f59f27 Mon Sep 17 00:00:00 2001 From: Andrei Aristarkhov Date: Wed, 30 Sep 2026 20:42:21 -0700 Subject: [PATCH 019/101] build(sbom): credit a staged directory, and the holders it was missing A third audit round, run against the regenerated wheel. The previous round's build-directory fix only half landed: the staged-component map still built its keys with a literal `build/` while the index it is matched against used the real directory name. The whole branch was dead in CI, and the MJCF-stripped assets it exists for came out attributed to nothing. Identical bytes still settle nothing, but the directory a member was staged into often does: it is already explained by exactly one of the candidates. MuJoCo's licence text, and the CloudXR libraries both SDKs ship byte-for-byte, take that component instead of naming every candidate -- which overstated what the others contribute. Saying the build recorded nothing was untrue where it plainly had. Checked-in third-party code was scanned for its copyright only where a compiler saw it, so a holder whose code reaches the wheel as Python was reported as contributing no file to a wheel that ships it. Every tracked file that can carry a REUSE header is scanned now, and a wheel member resolving to repository source credits the holder the scan found rather than dropping it. The notices file carries each component's copyright line. For one that ships no licence file of its own, the packaged reference text names no holder, so that line was the only notice the wheel had and it was in the SPDX document alone. Signed-off-by: Andrei Aristarkhov --- docs/source/references/sbom.rst | 7 +-- scripts/sbom/sbom/evidence.py | 18 ++++++-- scripts/sbom/sbom/inventory.py | 79 ++++++++++++++++++++++++++++----- scripts/sbom/sbom/report.py | 4 ++ 4 files changed, 90 insertions(+), 18 deletions(-) diff --git a/docs/source/references/sbom.rst b/docs/source/references/sbom.rst index b7fb792dbd..31d279bc8f 100644 --- a/docs/source/references/sbom.rst +++ b/docs/source/references/sbom.rst @@ -24,9 +24,10 @@ What a wheel carries └── sboms/ └── .spdx.json # SPDX 2.3 contents inventory -Every text there comes from this repository or from a component the wheel redistributes. -The SPDX reference texts the collector matches against are a separate thing and are never -packaged — see `How licenses are determined`_. +Every text there comes from this repository, or from a component the wheel redistributes, +or — for a component that ships no license file and states only an identifier in its source +headers — is the reference text that identifier names, beside the copyright line those +headers state. See `How licenses are determined`_. ``.dist-info/sboms/`` is what the `binary distribution format `_ diff --git a/scripts/sbom/sbom/evidence.py b/scripts/sbom/sbom/evidence.py index d67b9369a4..165d5b83f1 100644 --- a/scripts/sbom/sbom/evidence.py +++ b/scripts/sbom/sbom/evidence.py @@ -32,6 +32,9 @@ _walk, ) +# Suffixes whose files conventionally carry a REUSE header naming their holder. +_TAGGABLE_SOURCE = {".py", ".c", ".cc", ".cpp", ".h", ".hpp", ".pyi", ".sh", ".cmake"} + # Cache entries that pin what a wheel contains. The rest of the cache is paths on # the builder. _CACHE_KEYS = re.compile( @@ -211,7 +214,8 @@ def discover(repo_root: Path, build_dir: Path) -> Discovery: # from source, so a directory walk cannot, and a wheel member matching this # build's own output would be reported as a copy of repository source. repo_files = FileIndex() - repo_files.add_paths(repo_root, ".", tracked_files(repo_root)) + tracked = tracked_files(repo_root) + repo_files.add_paths(repo_root, ".", tracked) # _deps/*-src is the component-source domain, already indexed above; a wheel # member matching one is a copy of upstream, not something this build made. @@ -249,7 +253,7 @@ def discover(repo_root: Path, build_dir: Path) -> Discovery: if match: owners.add(match.split("/", 1)[0].removesuffix("-src")) if len(owners) == 1: - display = f"build/{root.relative_to(build_dir).as_posix()}" + display = f"{build_dir.name}/{root.relative_to(build_dir).as_posix()}" staged_component_dirs[display] = owners.pop() # Third-party code checked in here rather than fetched: the candidates are @@ -260,7 +264,15 @@ def discover(repo_root: Path, build_dir: Path) -> Discovery: if root.is_dir(): ownership.register_root(root, root.name.removesuffix("-src")) - candidates: set[Path] = set() + # Every tracked file that can carry a REUSE header, plus whatever the build + # compiled. Scanning only compiled sources missed a third party whose code + # reaches the wheel as Python: nothing compiles it, so nothing looked at it, + # and the component was reported as contributing no file. + candidates: set[Path] = { + repo_root / item + for item in tracked + if Path(item).suffix.lower() in _TAGGABLE_SOURCE + } for node in graph._nodes.values(): # noqa: SLF001 - same package candidates.update( source diff --git a/scripts/sbom/sbom/inventory.py b/scripts/sbom/sbom/inventory.py index 3349160c1e..0a7149d598 100644 --- a/scripts/sbom/sbom/inventory.py +++ b/scripts/sbom/sbom/inventory.py @@ -168,7 +168,7 @@ def resolve( # indexed first, and invent that component's license obligation with it. # The path is the only evidence such a file carries. if candidate_digest == EMPTY_SHA256: - return self._resolve_by_path(wheel_path, source, build_id) + return self._resolve_by_path(wheel_path, source, build_id, candidate_digest) artifacts = self._artifacts_by_hash.get(candidate_digest) if artifacts: @@ -182,7 +182,9 @@ def resolve( # whichever happened to be indexed first. shared = self._shared_content(candidate_digest) if shared is not None: - by_path = self._resolve_by_path(wheel_path, source, build_id) + by_path = self._resolve_by_path( + wheel_path, source, build_id, candidate_digest + ) if by_path is not None: return by_path return shared(wheel_path, source) @@ -210,9 +212,7 @@ def resolve( repo_file = self.repo_files.by_hash.get(candidate_digest) if repo_file: - return Attribution( - path=wheel_path, origin="repo-source", detail=f"{source}; {repo_file}" - ) + return self._from_repo_file(wheel_path, source, repo_file, f"{source}; ") build_file = self.build_files.by_hash.get(candidate_digest) if build_file: @@ -244,7 +244,7 @@ def resolve( if by_build_id is not None: return by_build_id - by_path = self._resolve_by_path(wheel_path, source, build_id) + by_path = self._resolve_by_path(wheel_path, source, build_id, candidate_digest) if by_path is not None: return by_path @@ -260,6 +260,25 @@ def resolve( return self._from_build_host(wheel_path, soname) return None + def _from_repo_file( + self, wheel_path: str, source: str, repo_file: str, lead: str + ) -> Attribution: + """A file this repository holds -- and, where it is not ours, whose. + + Checked-in third-party code states its own copyright, which is how the + vendored scan found it. Reporting the member as plain repository source + would drop that holder, and the component would be reported as + contributing nothing to a wheel that ships its file. + """ + owner = self.ownership.owner_of(self.repo_root / repo_file.removeprefix("./")) + return Attribution( + path=wheel_path, + origin="repo-source", + detail=f"{lead}{repo_file}", + components={owner: {"copied-file"}} if owner else {}, + primary=owner, + ) + def _staged_component_of(self, build_file: str) -> str | None: """The component whose tree this build staged under _deps//.""" for prefix, key in self.staged_component_dirs.items(): @@ -379,7 +398,11 @@ def _from_build_host_by_build_id( ) def _resolve_by_path( - self, wheel_path: str, source: str, build_id: str | None = None + self, + wheel_path: str, + source: str, + build_id: str | None = None, + digest: str | None = None, ) -> Attribution | None: """Post-processing changes bytes; the path still says where they came from. @@ -439,11 +462,9 @@ def _resolve_by_path( repo_file = self.repo_files.path_suffix_match(wheel_path) if repo_file: - return Attribution( - path=wheel_path, - origin="repo-source", - detail=f"{source}; derived from {repo_file}", - ) + same = self.repo_files.digest_of_display(repo_file) == digest + lead = f"{source}; {'identical to' if same else 'derived from'} " + return self._from_repo_file(wheel_path, source, repo_file, lead) build_file = self.build_files.path_suffix_match(wheel_path) if build_file: @@ -479,6 +500,8 @@ def build( attribution.copyright_text = _copyright_text(entry) attributions[entry.name] = attribution + _resolve_shared_by_sibling(attributions) + roles: dict[str, set[str]] = {} for attribution in attributions.values(): for key, how in attribution.components.items(): @@ -520,6 +543,38 @@ def _build_ids_disagree(left: str | None, right: str | None) -> bool: return bool(left and right and left != right) +def _resolve_shared_by_sibling(attributions: dict[str, Attribution]) -> None: + """Let a staged directory settle what identical bytes cannot. + + `libcloudxr.so` differs between two SDKs and resolves by build-id, but the + libraries beside it are byte-identical in both, and MuJoCo's licence text is + the same file every Apache-2.0 project ships. In each case the directory the + build staged the member into is already explained by exactly one of the + candidates, which is a record of where it came from -- so saying the build + kept none would be untrue, and naming every candidate overstates what the + others contribute. + """ + settled: dict[str, set[str]] = {} + for name, item in attributions.items(): + if item.primary: + settled.setdefault(name.rsplit("/", 1)[0], set()).add(item.primary) + + for name, item in attributions.items(): + if item.primary or item.origin != "shared-content": + continue + owners = settled.get(name.rsplit("/", 1)[0], set()) & set(item.components) + if len(owners) != 1: + continue + key = owners.pop() + item.origin = "copied" + item.detail = ( + f"{item.detail.split(';')[0]}; these bytes are held by more than one " + f"component, and the directory it is staged into is explained by {key}" + ) + item.components = {key: {"copied-file"}} + item.primary = key + + def _attribute( entry: Entry, wheel: WheelInfo, diff --git a/scripts/sbom/sbom/report.py b/scripts/sbom/sbom/report.py index 85de8ecf2c..c710ff7940 100644 --- a/scripts/sbom/sbom/report.py +++ b/scripts/sbom/sbom/report.py @@ -55,6 +55,10 @@ def notices_markdown(inventory: Inventory, dist_info: str, wheel_name: str) -> s lines.append(f"## {component.name}") lines.append("") lines.append(f"- License: `{component.license_concluded}`") + if component.copyright_text != "NOASSERTION": + # A reference text names no holder, so for a component that ships no + # licence file of its own this is the only notice in the wheel. + lines.append(f"- Copyright: {component.copyright_text}") if component.license_declared != component.license_concluded: lines.append(f"- Declared: `{component.license_declared}`") if component.homepage != "NOASSERTION": From 2c1f7438435d9e190889aa2a642e2b0e7518f06f Mon Sep 17 00:00:00 2001 From: Andrei Aristarkhov Date: Wed, 30 Sep 2026 20:43:38 -0700 Subject: [PATCH 020/101] test(sbom): give the synthetic project a fixed clock Making the generated project a git checkout left its commit timestamped by the wall clock. A commit hashes that timestamp, the collector records the commit, and the wheel carries the record -- so the reproducibility test comparing two builds of the same project passed only when both landed in the same second. It caught a real property; the fixture was the thing that was not reproducible. Signed-off-by: Andrei Aristarkhov --- tests/python/sbom/synth.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/python/sbom/synth.py b/tests/python/sbom/synth.py index 0c8f57265d..8935569e74 100644 --- a/tests/python/sbom/synth.py +++ b/tests/python/sbom/synth.py @@ -282,12 +282,17 @@ def _git_checkout(root: Path, initialise: bool = True) -> None: real repository. """ _write(root / ".gitignore", "build/\ndist/\n") + # Fixed identity *and* clock: a commit hashes its timestamp, the collector + # records the commit, and the wheel carries that record -- so a drifting + # clock makes two builds of the same project differ. env = { **os.environ, "GIT_AUTHOR_NAME": "synth", "GIT_AUTHOR_EMAIL": "synth@example.invalid", + "GIT_AUTHOR_DATE": "2026-01-01T00:00:00+00:00", "GIT_COMMITTER_NAME": "synth", "GIT_COMMITTER_EMAIL": "synth@example.invalid", + "GIT_COMMITTER_DATE": "2026-01-01T00:00:00+00:00", } run = lambda *args: subprocess.run( # noqa: E731 - local shorthand ["git", "-C", str(root), *args], check=True, capture_output=True, env=env From 61805c3167c5b801f812bf67c00c6d72e92be3b9 Mon Sep 17 00:00:00 2001 From: Andrei Aristarkhov Date: Wed, 30 Sep 2026 22:41:56 -0700 Subject: [PATCH 021/101] build(sbom): stop overstating what the build did to a file A fourth audit round. It confirmed the previous rounds' fixes landed and found two of them had landed only on one of the two paths that needed them. A wheel member whose bytes match a component's source was labelled derived whatever the bytes said, so a verbatim copy was published as modified -- with an "origin" digest equal to its own, and a notices line reading "copied in and modified by the build". The repository branch beside it already made the comparison; now both do. The LicenseRef comment in the document said every one of them matched no text in the SPDX corpus. One matched under an identifier the shipped grammar cannot validate, which the report had been taught to distinguish and the document -- the artifact consumers read -- had not. The README fallback found the licence section, used it to decide there was a grant, then hashed and packaged the whole file. marchingcubecpp's 81-byte grant shipped as 1899 bytes of build instructions and a screenshot link, and a real licence quoted in such a section would sink below the coverage threshold on the surrounding prose alone. The licence report shortened every version to twelve characters. That is a commit abbreviated; for `1.1.0-2build1.1` it produced `1.1.0-2build`, which reads as a complete Debian version and is not one. The manifest took its timestamp from the first wheel's, dating the set before the last wheel it binds existed, and skipped the duplicate-filename check the release path runs. Both paths merge the same way now. A dependency generating sources into its own build tree had them counted as this project's, which marks the target first-party and would leave a wheel member built from it with no component edge. And an archive was indexed from anywhere under the repository root, so one staged into an install prefix could have explained a wheel member -- the build accounting for itself. Signed-off-by: Andrei Aristarkhov --- scripts/sbom/sbom/build.py | 14 +++++++++++--- scripts/sbom/sbom/cli.py | 10 ++++++---- scripts/sbom/sbom/discovery.py | 19 ++++++++++++++++--- scripts/sbom/sbom/document.py | 13 +++++++++++-- scripts/sbom/sbom/evidence.py | 11 +++++++++-- scripts/sbom/sbom/inventory.py | 19 +++++++++++++++---- scripts/sbom/sbom/licensing.py | 8 +++++++- scripts/sbom/sbom/report.py | 14 +++++++++++++- tests/python/sbom/test_discovery.py | 8 ++++++-- 9 files changed, 94 insertions(+), 22 deletions(-) diff --git a/scripts/sbom/sbom/build.py b/scripts/sbom/sbom/build.py index 5d3395a2d7..11cba628de 100644 --- a/scripts/sbom/sbom/build.py +++ b/scripts/sbom/sbom/build.py @@ -242,9 +242,17 @@ def build( def merge_manifests(paths: list[Path]) -> dict: """Fold per-variant manifests into the one a release advertises.""" - wheels: list[dict] = [] - for path in sorted(paths): - wheels.extend(json.loads(path.read_text(encoding="utf-8"))["wheels"]) + return merge_records( + [ + item + for path in sorted(paths) + for item in json.loads(path.read_text(encoding="utf-8"))["wheels"] + ] + ) + + +def merge_records(wheels: list[dict]) -> dict: + """The manifest for a set of wheels, stamped when the set was complete.""" names = [item["filename"] for item in wheels] duplicates = sorted({name for name in names if names.count(name) > 1}) if duplicates: diff --git a/scripts/sbom/sbom/cli.py b/scripts/sbom/sbom/cli.py index 3718d544c8..f02d639e91 100644 --- a/scripts/sbom/sbom/cli.py +++ b/scripts/sbom/sbom/cli.py @@ -70,10 +70,12 @@ def _cmd_build(args: argparse.Namespace) -> int: ) print(f"packaged evidence into {wheel.name}") - merged = { - **manifests[0], - "wheels": [item for manifest in manifests for item in manifest["wheels"]], - } + # Through the same merge as a release: inheriting the first wheel's manifest + # dated the set before the last wheel it binds existed, and skipped the + # duplicate-filename check. + merged = build_module.merge_records( + [item for manifest in manifests for item in manifest["wheels"]] + ) manifest_path = out_dir / "manifest.json" manifest_path.write_text( json.dumps(merged, indent=2, sort_keys=True) + "\n", encoding="utf-8" diff --git a/scripts/sbom/sbom/discovery.py b/scripts/sbom/sbom/discovery.py index 4ba1e5db0a..029a9064cb 100644 --- a/scripts/sbom/sbom/discovery.py +++ b/scripts/sbom/sbom/discovery.py @@ -953,8 +953,15 @@ def tracked_files(repo_root: Path) -> list[str]: return [item for item in listing.split("\0") if item] -def discover_archives(repo_root: Path, skip: tuple[Path, ...] = ()) -> ArchiveIndex: - """Index archives held in the repository, not ones a build unpacked.""" +def discover_archives( + repo_root: Path, held: frozenset[str], skip: tuple[Path, ...] = () +) -> ArchiveIndex: + """Index archives held in the repository, not ones a build unpacked. + + `held` is what the repository tracks. It has no default: an empty set + silently indexes nothing, and a caller that forgot it would see every + archive-derived wheel member become unexplainable rather than wrong. + """ index = ArchiveIndex() skipped = tuple(item.resolve() for item in skip) for path in _walk(repo_root): @@ -963,7 +970,13 @@ def discover_archives(repo_root: Path, skip: tuple[Path, ...] = ()) -> ArchiveIn name = path.name.lower() if not name.endswith(ARCHIVE_SUFFIXES): continue - index.add_archive(path, path.relative_to(repo_root).as_posix()) + relative = path.relative_to(repo_root).as_posix() + # Held here, or fetched by this build and recorded as such. An archive + # anywhere else is output -- staged into an install prefix, say -- and + # letting it explain a wheel member is the build explaining itself. + if relative not in held and not path.with_name(path.name + ".source").is_file(): + continue + index.add_archive(path, relative) return index diff --git a/scripts/sbom/sbom/document.py b/scripts/sbom/sbom/document.py index b426bc6aaf..2051af6758 100644 --- a/scripts/sbom/sbom/document.py +++ b/scripts/sbom/sbom/document.py @@ -109,8 +109,17 @@ def _extracted_licenses(components: list[Component]) -> list[dict]: "name": f"License terms shipped with {component.name}", "extractedText": "\n\n".join(item.text for item in grants), "comment": ( - "Matched no text in the SPDX reference corpus; reproduced here " - "verbatim from " + ", ".join(item.path for item in grants) + "." + ( + "Matched no text in the SPDX reference corpus" + if not any(item.matches for item in grants) + # Matched, but under an id the grammar shipped with this + # document cannot validate, so it cannot be named here. + else "Matched an identifier newer than the SPDX grammar " + "this document ships with" + ) + + "; reproduced here verbatim from " + + ", ".join(item.path for item in grants) + + "." ), } return [entries[key] for key in sorted(entries)] diff --git a/scripts/sbom/sbom/evidence.py b/scripts/sbom/sbom/evidence.py index 165d5b83f1..e48cb92989 100644 --- a/scripts/sbom/sbom/evidence.py +++ b/scripts/sbom/sbom/evidence.py @@ -201,8 +201,9 @@ def discover(repo_root: Path, build_dir: Path) -> Discovery: ), ) + tracked = tracked_files(repo_root) components = discover_source_trees(deps_dir) - archives = discover_archives(repo_root, skip=not_source) + archives = discover_archives(repo_root, frozenset(tracked), skip=not_source) components.update(_archive_components(archives, repo_root)) source_files = FileIndex() @@ -214,7 +215,6 @@ def discover(repo_root: Path, build_dir: Path) -> Discovery: # from source, so a directory walk cannot, and a wheel member matching this # build's own output would be reported as a copy of repository source. repo_files = FileIndex() - tracked = tracked_files(repo_root) repo_files.add_paths(repo_root, ".", tracked) # _deps/*-src is the component-source domain, already indexed above; a wheel @@ -263,6 +263,13 @@ def discover(repo_root: Path, build_dir: Path) -> Discovery: for root in sorted(deps_dir.glob("*-src")): if root.is_dir(): ownership.register_root(root, root.name.removesuffix("-src")) + # A dependency generates sources into its own build tree -- glfw's Wayland + # protocol headers, Catch2's generated config. Those sit under the build + # directory and belong to no *-src checkout, so without this they count as + # this project's own sources and the target reads as first-party. + for root in sorted(deps_dir.glob("*-build")): + if root.is_dir(): + ownership.register_root(root, root.name.removesuffix("-build")) # Every tracked file that can carry a REUSE header, plus whatever the build # compiled. Scanning only compiled sources missed a third party whose code diff --git a/scripts/sbom/sbom/inventory.py b/scripts/sbom/sbom/inventory.py index 0a7149d598..5337b19e9b 100644 --- a/scripts/sbom/sbom/inventory.py +++ b/scripts/sbom/sbom/inventory.py @@ -449,14 +449,25 @@ def _resolve_by_path( if component_file: key = component_file.split("/", 1)[0].removesuffix("-src") origin_digest = self.source_files.digest_of_display(component_file) + # Reaching this branch means the content matched more than one place, + # not that anything transformed the file. Where the input's bytes are + # these bytes it is a copy, and quoting an origin digest equal to the + # member's own would say nothing. + same = origin_digest == digest return Attribution( path=wheel_path, - origin="derived", + origin="copied" if same else "derived", detail=( - f"{source}; derived from {component_file}" - + (f" (origin sha256:{origin_digest})" if origin_digest else "") + f"{source}; " + + ("identical to " if same else "derived from ") + + component_file + + ( + "" + if same or not origin_digest + else f" (origin sha256:{origin_digest})" + ) ), - components={key: {"derived-file"}}, + components={key: {"copied-file" if same else "derived-file"}}, primary=key, ) diff --git a/scripts/sbom/sbom/licensing.py b/scripts/sbom/sbom/licensing.py index 7e6b12459b..d1919de9ae 100644 --- a/scripts/sbom/sbom/licensing.py +++ b/scripts/sbom/sbom/licensing.py @@ -383,8 +383,14 @@ def _readme_fallback( section = _README_LICENSE_SECTION.search(text) if section is None or not section.group("body").strip(): continue + # The section, not the whole README. Hashing the file would publish its + # build instructions as the terms that shipped, and would bury a real + # licence quoted there under enough prose to fall below the coverage + # threshold and degrade to a LicenseRef. display = f"{display_prefix}/{entry.name}" - return [_evidence(component, "readme-section", display, text, "grant")] + return [ + _evidence(component, "readme-section", display, section.group(0), "grant") + ] return [] diff --git a/scripts/sbom/sbom/report.py b/scripts/sbom/sbom/report.py index c710ff7940..76a4f00329 100644 --- a/scripts/sbom/sbom/report.py +++ b/scripts/sbom/sbom/report.py @@ -24,6 +24,18 @@ _SAFE_NAME = re.compile(r"[^A-Za-z0-9._-]") +_COMMIT = re.compile(r"^[0-9a-f]{40}$") + + +def _short(version: str) -> str: + """Abbreviate a commit; leave anything else whole. + + Truncating to a fixed width turned `1.1.0-2build1.1` into `1.1.0-2build`, + which reads as a complete Debian version and is not one. + """ + return version[:12] if _COMMIT.match(version) else version + + def packaged_license_path(dist_info: str, component_key: str, evidence) -> str: """Where one component's license text lands inside the wheel. @@ -181,7 +193,7 @@ def report_markdown(payload: dict) -> str: ) version = item["version"] lines.append( - f"| {item['name']} | `{version[:12]}` | `{item['license_concluded']}` | " + f"| {item['name']} | `{_short(version)}` | `{item['license_concluded']}` | " f"{how or '—'} | {evidence} |" ) diff --git a/tests/python/sbom/test_discovery.py b/tests/python/sbom/test_discovery.py index 5fc47fff66..cc399d0969 100644 --- a/tests/python/sbom/test_discovery.py +++ b/tests/python/sbom/test_discovery.py @@ -95,7 +95,9 @@ def test_a_test_only_dependency_reaches_no_shipped_artifact(workspace): def test_archive_members_are_indexed_by_content(workspace): - index = discovery.discover_archives(workspace.root) + index = discovery.discover_archives( + workspace.root, frozenset(discovery.tracked_files(workspace.root)) + ) assert synth.SDK_ARCHIVE in index.archives demo = index.by_name["libdemo.so"] @@ -104,7 +106,9 @@ def test_archive_members_are_indexed_by_content(workspace): def test_archive_version_and_license_come_from_the_archive(workspace): - index = discovery.discover_archives(workspace.root) + index = discovery.discover_archives( + workspace.root, frozenset(discovery.tracked_files(workspace.root)) + ) record = index.archives[synth.SDK_ARCHIVE] assert record["version_text"] == "1.2.0" From 7858c673c5815818a08c2dd0c32030c933649e1f Mon Sep 17 00:00:00 2001 From: Andrei Aristarkhov Date: Wed, 30 Sep 2026 23:12:57 -0700 Subject: [PATCH 022/101] build(sbom): verify the wheel, not what the wheel says about itself Two ways a wheel could pass `check` while shipping a file the document does not account for. Both reproduced against a real wheel before and after. The verifier took its excluded-files list out of the document it was verifying and used it to suppress the coverage check, asserting only that the embedded SBOM appeared in it. Adding a module, listing it as excluded and regenerating RECORD produced a wheel that `check` called OK -- the command published to consumers and run by CI. The list is exactly RECORD and the document; anything else fails, and the check proceeds against the permitted set. Tracing an origin starts from the staged file where there is one, which is right: that is where the member came from. The member's own digest was then never looked at again, so a member that differed from its staged twin was still described as byte-identical to whatever explained the staged bytes, and the member set that the gate requires every file of to be explained was effectively the staged tree's. Where they differ the member was rewritten after staging, and is reported as derived rather than as a copy -- which is what the reference already said should happen. Whether a member is an exact copy is now carried as its own fact rather than inferred from the origin label. Inferring it meant a byte-identical file could be COPY_OF or GENERATED_FROM depending on which index happened to hold the bytes. Signed-off-by: Andrei Aristarkhov --- scripts/sbom/sbom/document.py | 22 ++---------- scripts/sbom/sbom/inventory.py | 62 ++++++++++++++++++++++++++++++--- scripts/sbom/sbom/validate.py | 26 ++++++++++++-- tests/python/sbom/test_gates.py | 26 ++++++++++++++ 4 files changed, 109 insertions(+), 27 deletions(-) diff --git a/scripts/sbom/sbom/document.py b/scripts/sbom/sbom/document.py index 2051af6758..27e4fded3e 100644 --- a/scripts/sbom/sbom/document.py +++ b/scripts/sbom/sbom/document.py @@ -7,13 +7,11 @@ import hashlib import re -import os -from datetime import datetime, timezone from packageurl import PackageURL from packaging.requirements import Requirement -from . import SPDX_VERSION, TOOL_NAME, TOOL_VERSION, licensing +from . import SPDX_VERSION, TOOL_NAME, TOOL_VERSION, licensing, stamped_now from .discovery import Component from .inventory import Inventory from .wheelfile import WheelInfo @@ -84,15 +82,6 @@ def _component_package(component: Component, roles: set[str]) -> dict: return package -def _file_license_refs(inventory: Inventory) -> set[str]: - """LicenseRef- identifiers that arrive through a file's own REUSE tag.""" - refs: set[str] = set() - for attribution in inventory.attributions.values(): - if attribution.spdx_tag: - refs.update(licensing.license_refs(attribution.spdx_tag)) - return refs - - def _extracted_licenses(components: list[Component]) -> list[dict]: """Texts for every LicenseRef the document uses, taken from the shipped file.""" entries: dict[str, dict] = {} @@ -293,7 +282,7 @@ def build_document( if name not in file_ids: continue attribution = inventory.attributions[name] - exact = attribution.origin in {"copied", "archive-copy"} + exact = attribution.exact relationships.append( { "spdxElementId": file_ids[name], @@ -392,12 +381,7 @@ def build_document( # SOURCE_DATE_EPOCH makes the whole wheel reproducible, which is what lets a # rebuild be compared against a published one byte for byte. - epoch = os.environ.get("SOURCE_DATE_EPOCH") - stamp = ( - datetime.fromtimestamp(int(epoch), timezone.utc) - if epoch and epoch.isdigit() - else datetime.now(timezone.utc) - ) + stamp = stamped_now() return { "spdxVersion": SPDX_VERSION, diff --git a/scripts/sbom/sbom/inventory.py b/scripts/sbom/sbom/inventory.py index 5337b19e9b..b35789658c 100644 --- a/scripts/sbom/sbom/inventory.py +++ b/scripts/sbom/sbom/inventory.py @@ -58,6 +58,11 @@ class Attribution: primary: str | None = None spdx_tag: str | None = None copyright_text: str | None = None + # Whether the member is those bytes, as opposed to something made from them. + # Kept apart from `origin`, which says where it came from: inferring one from + # the other made the same fact answerable two ways depending on which index + # happened to hold the bytes. + exact: bool = False def as_json(self) -> dict: return { @@ -159,10 +164,42 @@ def resolve( """Explain one member: by content, then by path, then by the machine.""" candidate_digest = digest source = "wheel member" + patched = False if probe is not None and probe.is_file(): candidate_digest = sha256_file(probe) + # The staged file is what the build put there; the member is what the + # wheel ships. Tracing from the staged bytes finds the origin, but + # the two can differ -- a repair tool rewrites members after staging + # -- and saying the member is a copy of something it does not hash to + # would describe a file the wheel does not contain. + patched = candidate_digest != digest source = f"staged at {probe.relative_to(self.build_dir).as_posix()}" + found = self._trace( + wheel_path, candidate_digest, digest, source, soname, build_id + ) + if found is not None and patched: + found.origin = { + "copied": "derived", + "archive-copy": "archive-derived", + }.get(found.origin, found.origin) + found.exact = False + found.detail += ( + "; the member differs from that staged file, so it was rewritten " + "after staging" + ) + return found + + def _trace( + self, + wheel_path: str, + candidate_digest: str, + digest: str, + source: str, + soname: str | None, + build_id: str | None, + ) -> Attribution | None: + """Find what explains these bytes, in order of how much it proves.""" # Zero bytes are identical everywhere, so a content match proves nothing: # it would hand an empty marker file to whichever component happened to be # indexed first, and invent that component's license obligation with it. @@ -195,6 +232,7 @@ def resolve( return Attribution( path=wheel_path, origin="copied", + exact=True, detail=f"{source}; byte-identical to {component_file}", components={key: {"copied-file"}}, primary=key, @@ -205,6 +243,7 @@ def resolve( return Attribution( path=wheel_path, origin="archive-copy", + exact=True, detail=f"{source}; byte-identical to {member.path} in {member.container}", components={member.container: {"extracted-file"}}, primary=member.container, @@ -212,7 +251,9 @@ def resolve( repo_file = self.repo_files.by_hash.get(candidate_digest) if repo_file: - return self._from_repo_file(wheel_path, source, repo_file, f"{source}; ") + return self._from_repo_file( + wheel_path, source, repo_file, f"{source}; ", True + ) build_file = self.build_files.by_hash.get(candidate_digest) if build_file: @@ -261,7 +302,7 @@ def resolve( return None def _from_repo_file( - self, wheel_path: str, source: str, repo_file: str, lead: str + self, wheel_path: str, source: str, repo_file: str, lead: str, same: bool ) -> Attribution: """A file this repository holds -- and, where it is not ours, whose. @@ -274,8 +315,11 @@ def _from_repo_file( return Attribution( path=wheel_path, origin="repo-source", + exact=same, detail=f"{lead}{repo_file}", - components={owner: {"copied-file"}} if owner else {}, + components={owner: {"copied-file" if same else "derived-file"}} + if owner + else {}, primary=owner, ) @@ -352,7 +396,13 @@ def _resolve_by_build_id( self, wheel_path: str, source: str, build_id: str ) -> Attribution | None: """The pristine bytes this member was patched from, named by build-id.""" - member = self.archives.by_build_id.get(build_id) + # The same library can sit in two archives under two sets of terms, and + # these two SDKs share five build-ids. Every neighbouring index refuses + # to name one of several candidates; this one must too, or a patched + # member takes whichever archive happened to be walked first. + members = self.archives.all_by_build_id.get(build_id, []) + containers = {item.container for item in members} + member = members[0] if len(containers) == 1 else None if member: return Attribution( path=wheel_path, @@ -457,6 +507,7 @@ def _resolve_by_path( return Attribution( path=wheel_path, origin="copied" if same else "derived", + exact=same, detail=( f"{source}; " + ("identical to " if same else "derived from ") @@ -475,7 +526,7 @@ def _resolve_by_path( if repo_file: same = self.repo_files.digest_of_display(repo_file) == digest lead = f"{source}; {'identical to' if same else 'derived from'} " - return self._from_repo_file(wheel_path, source, repo_file, lead) + return self._from_repo_file(wheel_path, source, repo_file, lead, same) build_file = self.build_files.path_suffix_match(wheel_path) if build_file: @@ -578,6 +629,7 @@ def _resolve_shared_by_sibling(attributions: dict[str, Attribution]) -> None: continue key = owners.pop() item.origin = "copied" + item.exact = True item.detail = ( f"{item.detail.split(';')[0]}; these bytes are held by more than one " f"component, and the directory it is staged into is explained by {key}" diff --git a/scripts/sbom/sbom/validate.py b/scripts/sbom/sbom/validate.py index 05b5673e46..efa4496ab9 100644 --- a/scripts/sbom/sbom/validate.py +++ b/scripts/sbom/sbom/validate.py @@ -113,8 +113,17 @@ def verify_wheel( "packageVerificationCodeExcludedFiles", [] ) } - if sbom_name not in excluded: - failures.append("the embedded SBOM must be listed among the excluded files") + # Exactly these two, not "at least". The list lives in the document being + # checked, so treating it as a set of files to skip lets anyone who can edit + # the document exempt anything they add to the wheel from the coverage check + # below -- which is the whole of what this command promises a consumer. + permitted = {f"{wheel.dist_info}/RECORD", sbom_name} + if excluded != permitted: + failures.append( + "the excluded-files list must be exactly " + f"{sorted(permitted)}, not {sorted(excluded)}" + ) + excluded = permitted documented = {item["fileName"].removeprefix("./"): item for item in spdx["files"]} actual = {item.name: item for item in wheel.entries} @@ -178,6 +187,12 @@ def _check_record(wheel_path: Path, wheel: wheelfile.WheelInfo) -> list[str]: def _check_attribution(evidence_doc: dict, wheel: wheelfile.WheelInfo) -> list[str]: """Every wheel member must name where it came from, and every shipped component must have had a license text to read.""" + stated = evidence_doc.get("wheel") + if stated and stated != wheel.path.name: + # Checking a wheel against another wheel's evidence compares two unrelated + # member sets; the mismatch it produces describes the wrong problem. + return [f"build evidence describes {stated}, not {wheel.path.name}"] + failures: list[str] = [] attributed = {item["path"] for item in evidence_doc.get("attributions", [])} @@ -190,7 +205,12 @@ def _check_attribution(evidence_doc: dict, wheel: wheelfile.WheelInfo) -> list[s for key, component in evidence_doc.get("components", {}).items(): if not component.get("in_this_wheel"): continue - if not any(item["kind"] == "grant" for item in component.get("evidence", [])): + # The same test the build gate applies: a REUSE pool is a real text even + # though it names no single expression, so publishing on one and then + # failing verification for want of a grant would contradict the gate. + if not any( + item["kind"] in ("grant", "pool") for item in component.get("evidence", []) + ): failures.append( f"{key} is redistributed but no license text was found for it" ) diff --git a/tests/python/sbom/test_gates.py b/tests/python/sbom/test_gates.py index 85dd0fa8c5..ee21e1ac7a 100644 --- a/tests/python/sbom/test_gates.py +++ b/tests/python/sbom/test_gates.py @@ -397,3 +397,29 @@ def test_bytes_more_than_one_component_holds_name_no_single_source( assert "more than one component" in entry["comment"] # Both candidates named, neither claimed as the source. assert "alpha" in entry["comment"] and "beta" in entry["comment"] + + +def test_the_excluded_list_cannot_exempt_a_smuggled_file(built): + """The list lives in the document being checked. + + Treating it as files to skip lets whoever edits the document exempt anything + they add to the wheel from the only coverage check a consumer runs. + """ + with zipfile.ZipFile(built["wheel"]) as archive: + payload = {item: archive.read(item) for item in archive.namelist()} + + payload["isaaccapture/_backdoor.py"] = b"import os\n" + document = f"{DIST_INFO}/sboms/{WHEEL_NAME}.spdx.json" + spdx = json.loads(payload[document]) + for package in spdx["packages"]: + if package["SPDXID"] == "SPDXRef-Package-wheel": + package["packageVerificationCode"][ + "packageVerificationCodeExcludedFiles" + ].append("./isaaccapture/_backdoor.py") + payload[document] = json.dumps(spdx, indent=2, sort_keys=True).encode() + b"\n" + synth.write_wheel(built["wheel"], payload) + + failures = validate_module.check(built["wheel"]) + + assert any("excluded-files list must be exactly" in item for item in failures) + assert any("not covered by the SBOM" in item for item in failures) From 8759e7bff6103502214eb7a880d474f15caa553e Mon Sep 17 00:00:00 2001 From: Andrei Aristarkhov Date: Wed, 30 Sep 2026 23:13:31 -0700 Subject: [PATCH 023/101] build(sbom): give every index and every artifact the same rules The rest of the fifth audit round, all of it one emitter or index behaving unlike the one beside it. Every content index pairs a first-wins map with a check for whether more than one component holds the bytes. The build-id index had none, so a patched library took whichever archive was walked first -- and the two CloudXR SDKs share five build-ids under two different sets of terms. That is the exact case the build-id route exists to serve, masked today only because those five still match by content and never reach it. SOURCE_DATE_EPOCH was honoured by one of the four things that stamp a time, so a rebuild could be compared to a published wheel but not to its published evidence, which is the chain the manifest binds. One clock now, in one place. Verification accepted build evidence for a different wheel, comparing two unrelated member sets and reporting the mismatch as missing attributions. It checks the name the evidence records, the way the manifest check beside it already did. The publication gate counts a REUSE pool as a licence text and verification did not, so a component could publish and then fail the check CI runs with a message saying no text was found. And the build warned about one of the two gap lists it carries, hiding the stricter one. Removes `_file_license_refs`, dead since it was written and not completable: a file tag naming a LicenseRef has no text to define it with, and an undefined reference is caught at check time. Signed-off-by: Andrei Aristarkhov --- scripts/sbom/sbom/__init__.py | 16 ++++++++++++++++ scripts/sbom/sbom/build.py | 7 +++---- scripts/sbom/sbom/cli.py | 15 +++++++++++---- scripts/sbom/sbom/discovery.py | 2 ++ scripts/sbom/sbom/evidence.py | 5 ++--- 5 files changed, 34 insertions(+), 11 deletions(-) diff --git a/scripts/sbom/sbom/__init__.py b/scripts/sbom/sbom/__init__.py index 073da5fe74..1c53e55367 100644 --- a/scripts/sbom/sbom/__init__.py +++ b/scripts/sbom/sbom/__init__.py @@ -7,3 +7,19 @@ # Bump on any change that alters the shape of an emitted document or report. TOOL_VERSION = "1.0.0" SPDX_VERSION = "SPDX-2.3" + + +def stamped_now(): + """The time to publish, honouring SOURCE_DATE_EPOCH. + + Every artifact a release advertises uses this. Having only the SPDX document + honour the epoch made a rebuild comparable to a published wheel but not to + its published evidence, which is the chain the manifest binds. + """ + import os + from datetime import datetime, timezone + + epoch = os.environ.get("SOURCE_DATE_EPOCH") + if epoch and epoch.isdigit(): + return datetime.fromtimestamp(int(epoch), timezone.utc) + return datetime.now(timezone.utc) diff --git a/scripts/sbom/sbom/build.py b/scripts/sbom/sbom/build.py index 11cba628de..1e34848a37 100644 --- a/scripts/sbom/sbom/build.py +++ b/scripts/sbom/sbom/build.py @@ -11,10 +11,9 @@ import shutil import tempfile from dataclasses import replace -from datetime import datetime, timezone from pathlib import Path -from . import TOOL_NAME, TOOL_VERSION +from . import TOOL_NAME, TOOL_VERSION, stamped_now from . import document as document_module from . import evidence as evidence_module from . import licensing @@ -213,7 +212,7 @@ def build( # touched again no longer matches its own manifest entry. return { "schema": "isaaccapture-sbom-manifest/1", - "generated_at": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"), + "generated_at": stamped_now().strftime("%Y-%m-%dT%H:%M:%SZ"), "tool": {"name": TOOL_NAME, "version": TOOL_VERSION}, "wheels": [ { @@ -259,7 +258,7 @@ def merge_records(wheels: list[dict]) -> dict: raise BuildError(f"the same wheel filename was published twice: {duplicates}") return { "schema": "isaaccapture-sbom-manifest/1", - "generated_at": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"), + "generated_at": stamped_now().strftime("%Y-%m-%dT%H:%M:%SZ"), "tool": {"name": TOOL_NAME, "version": TOOL_VERSION}, "wheels": sorted(wheels, key=lambda item: item["filename"]), } diff --git a/scripts/sbom/sbom/cli.py b/scripts/sbom/sbom/cli.py index f02d639e91..5e0e5b9879 100644 --- a/scripts/sbom/sbom/cli.py +++ b/scripts/sbom/sbom/cli.py @@ -83,10 +83,17 @@ def _cmd_build(args: argparse.Namespace) -> int: print(f"manifest: {manifest_path}") for item in merged["wheels"]: - for entry in item["components_with_unidentified_license"]: - print( - f"::warning::{item['filename']}: {entry['component']}: {entry['reason']}" - ) + # Both gap lists. They sit in the same record and mean the same kind of + # thing to a reader; surfacing one and not the other hid the stricter one. + for key in ( + "components_without_license_evidence", + "components_with_unidentified_license", + ): + for entry in item[key]: + print( + f"::warning::{item['filename']}: {entry['component']}: " + f"{entry['reason']}" + ) return 0 diff --git a/scripts/sbom/sbom/discovery.py b/scripts/sbom/sbom/discovery.py index 029a9064cb..59a42dfd23 100644 --- a/scripts/sbom/sbom/discovery.py +++ b/scripts/sbom/sbom/discovery.py @@ -858,6 +858,7 @@ def __init__(self) -> None: # Survives the RPATH/SONAME rewrite a repair tool applies on the way in, # so a patched wheel member still points at the archive it came from. self.by_build_id: dict[str, Member] = {} + self.all_by_build_id: dict[str, list[Member]] = {} self.archives: dict[str, dict] = {} def add_archive(self, path: Path, display: str) -> None: @@ -879,6 +880,7 @@ def add_archive(self, path: Path, display: str) -> None: if build_id: member = replace(member, build_id=build_id) self.by_build_id.setdefault(build_id, member) + self.all_by_build_id.setdefault(build_id, []).append(member) self.by_hash.setdefault(member.sha256, member) self.all_by_hash.setdefault(member.sha256, []).append(member) self.by_name.setdefault(Path(member.path).name, []).append(member) diff --git a/scripts/sbom/sbom/evidence.py b/scripts/sbom/sbom/evidence.py index e48cb92989..ebcc9cd5dc 100644 --- a/scripts/sbom/sbom/evidence.py +++ b/scripts/sbom/sbom/evidence.py @@ -8,13 +8,12 @@ import platform import re from dataclasses import dataclass -from datetime import datetime, timezone from pathlib import Path from urllib.parse import urlparse from packageurl import PackageURL -from . import TOOL_NAME, TOOL_VERSION, licensing +from . import TOOL_NAME, TOOL_VERSION, licensing, stamped_now from .discovery import ( ARCHIVE_SUFFIXES, ArchiveIndex, @@ -356,7 +355,7 @@ def document(discovery: Discovery, wheel_name: str, inventory) -> dict: return { "schema": "isaaccapture-build-evidence/2", "tool": {"name": TOOL_NAME, "version": TOOL_VERSION}, - "collected_at": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"), + "collected_at": stamped_now().strftime("%Y-%m-%dT%H:%M:%SZ"), "wheel": wheel_name, "project": { "commit": git(repo_root, "rev-parse", "HEAD"), From 27344d359e8dc854ed57d4dc1a90d0d1cec2e2d5 Mon Sep 17 00:00:00 2001 From: Andrei Aristarkhov Date: Wed, 30 Sep 2026 23:44:46 -0700 Subject: [PATCH 024/101] build(sbom): close the ways a wheel could lie about itself and pass A sixth round, run adversarially against the gates rather than over the document. Four of the attacks it built got through; all four are reproduced here before and after. A zip may carry two members under one name. Reading by name returns the last, so the first was hashed by nothing, compared to RECORD by nothing and reported as uncovered by nothing -- while which one a consumer extracts is up to their unzip. Inserting a second `isaaccapture/__init__.py` into the published wheel, touching nothing else, produced an archive all three gates called OK. Members are read by handle now, and an archive naming one twice is refused. The rewrite step is guarded too, since adding a member that already exists would create the same condition. Anything under `.dist-info/` was asserted to be metadata the build wrote, with no probe of any kind -- the one route around the gate every other member passes. A library dropped in there was published with a fabricated origin and never reached the licence pipeline at all. Only the names a packaging backend and this collector actually write are metadata; the rest is explained like everything else or it stops the build. `check-set` -- the release gate, and the command the reference tells a consumer to run over a download -- never read the build evidence whose digest it verifies, making it strictly weaker than the per-wheel check beside it. The document states a digest for every licence text it read, and the wheel packages those bytes; nothing compared them. A proprietary EULA could be swapped for Apache-2.0 and re-sealed, leaving one archive making four statements about a component with three contradicting the fourth. The document now records where it packaged each text, and verification reads that path -- by path, because two components can ship identical texts and a set membership test survives a swap inside one of them. Signed-off-by: Andrei Aristarkhov --- scripts/sbom/sbom/cli.py | 3 ++ scripts/sbom/sbom/document.py | 13 ++--- scripts/sbom/sbom/inventory.py | 34 ++++++++++--- scripts/sbom/sbom/report.py | 5 +- scripts/sbom/sbom/validate.py | 87 ++++++++++++++++++++++++++++++--- scripts/sbom/sbom/wheelfile.py | 26 +++++++++- tests/python/sbom/test_gates.py | 25 ++++++++++ 7 files changed, 173 insertions(+), 20 deletions(-) diff --git a/scripts/sbom/sbom/cli.py b/scripts/sbom/sbom/cli.py index 5e0e5b9879..63c46cbe7d 100644 --- a/scripts/sbom/sbom/cli.py +++ b/scripts/sbom/sbom/cli.py @@ -14,6 +14,7 @@ from . import TOOL_NAME, TOOL_VERSION from . import build as build_module from . import discovery +from . import wheelfile from . import evidence as evidence_module from . import licensing from . import validate as validate_module @@ -192,6 +193,8 @@ def main(argv: list[str] | None = None) -> int: build_module.BuildError, licensing.CorpusError, discovery.ArchiveError, + wheelfile.WheelError, + validate_module.ManifestError, ) as error: print(f"error: {error}", file=sys.stderr) return 2 diff --git a/scripts/sbom/sbom/document.py b/scripts/sbom/sbom/document.py index 27e4fded3e..82f29dbe85 100644 --- a/scripts/sbom/sbom/document.py +++ b/scripts/sbom/sbom/document.py @@ -11,7 +11,7 @@ from packageurl import PackageURL from packaging.requirements import Requirement -from . import SPDX_VERSION, TOOL_NAME, TOOL_VERSION, licensing, stamped_now +from . import SPDX_VERSION, TOOL_NAME, TOOL_VERSION, licensing, report, stamped_now from .discovery import Component from .inventory import Inventory from .wheelfile import WheelInfo @@ -33,10 +33,11 @@ def verification_code(sha1_digests: list[str]) -> str: return hashlib.sha1("".join(sorted(sha1_digests)).encode("ascii")).hexdigest() # noqa: S324 -def _attribution_texts(component: Component) -> list[str]: +def _attribution_texts(component: Component, dist_info: str) -> list[str]: texts = [ f"License evidence: {item.path} (sha256:{item.sha256}, {item.size} bytes, " - f"{item.origin}, {item.kind})" + f"{item.origin}, {item.kind}) packaged at " + f"{report.packaged_license_path(dist_info, component.key, item)}" + ( " identified as " + "; ".join( @@ -56,7 +57,7 @@ def _attribution_texts(component: Component) -> list[str]: return texts -def _component_package(component: Component, roles: set[str]) -> dict: +def _component_package(component: Component, roles: set[str], dist_info: str) -> dict: package = { "SPDXID": _spdx_id("Package", component.key), "name": component.name, @@ -74,7 +75,7 @@ def _component_package(component: Component, roles: set[str]) -> dict: "referenceLocator": component.purl, } ], - "attributionTexts": _attribution_texts(component), + "attributionTexts": _attribution_texts(component, dist_info), "sourceInfo": f"{component.source_info} Reaches this wheel as: {', '.join(sorted(roles))}.", } if component.homepage != NOASSERTION: @@ -256,7 +257,7 @@ def build_document( for key, component in inventory.components_present.items(): roles = inventory.component_roles.get(key, set()) - package = _component_package(component, roles) + package = _component_package(component, roles, wheel.dist_info) packages.append(package) package_id = package["SPDXID"] diff --git a/scripts/sbom/sbom/inventory.py b/scripts/sbom/sbom/inventory.py index b35789658c..4908d82613 100644 --- a/scripts/sbom/sbom/inventory.py +++ b/scripts/sbom/sbom/inventory.py @@ -12,6 +12,7 @@ from __future__ import annotations from dataclasses import dataclass, field +import re from pathlib import Path from typing import TYPE_CHECKING @@ -638,6 +639,12 @@ def _resolve_shared_by_sibling(attributions: dict[str, Attribution]) -> None: item.primary = key +_DIST_INFO_WRITTEN = re.compile( + r"^(METADATA|WHEEL|RECORD|INSTALLER|REQUESTED|entry_points\.txt|top_level\.txt" + r"|direct_url\.json|zip-safe|namespace_packages\.txt)$|^(licenses|sboms)/" +) + + def _attribute( entry: Entry, wheel: WheelInfo, @@ -645,11 +652,18 @@ def _attribute( staged_root: Path | None, ) -> Attribution | None: if entry.name.startswith(f"{wheel.dist_info}/"): - return Attribution( - path=entry.name, - origin="metadata", - detail="distribution metadata written by the build", - ) + relative = entry.name[len(wheel.dist_info) + 1 :] + # Only what a packaging backend and this collector actually write there. + # Treating the whole directory as metadata asserted an origin for + # anything placed in it, which is the one route around the gate that + # every other member has to pass. + if _DIST_INFO_WRITTEN.match(relative): + return Attribution( + path=entry.name, + origin="metadata", + exact=True, + detail="distribution metadata written by the build", + ) probe = staged_root / entry.name if staged_root is not None else None return resolver.resolve( @@ -675,4 +689,12 @@ def _spdx_tag(entry: Entry) -> str | None: # document as licenseConcluded, where an invalid expression fails the # published SBOM's own conformance check. normalized = licensing.normalized_expression(tag) - return None if normalized == "NOASSERTION" else normalized + if normalized == "NOASSERTION": + return None + # A LicenseRef is only meaningful alongside its text, and a file tag carries + # none: naming one here produced a document that failed its own check for an + # undefined reference, on a wheel the build had already rewritten and would + # refuse to process again. + if licensing.license_refs(normalized): + return None + return normalized diff --git a/scripts/sbom/sbom/report.py b/scripts/sbom/sbom/report.py index 76a4f00329..58e22beadb 100644 --- a/scripts/sbom/sbom/report.py +++ b/scripts/sbom/sbom/report.py @@ -93,7 +93,7 @@ def report(inventory: Inventory, evidence_doc: dict, wheel_name: str) -> dict: unidentified = [] for key, component in inventory.components_present.items(): - grants = [item for item in component.evidence if item.kind == "grant"] + grants = [item for item in component.evidence if item.kind in ("grant", "pool")] components.append( { "component": key, @@ -113,6 +113,9 @@ def report(inventory: Inventory, evidence_doc: dict, wheel_name: str) -> dict: "evidence": [item.as_json() for item in component.evidence], } ) + # Same test as the build gate and the verifier: a REUSE pool is a real + # text. Requiring a grant here advertised "no licence evidence" for a + # component the gates deliberately accepted on the pool they packaged. if not grants: no_evidence.append( { diff --git a/scripts/sbom/sbom/validate.py b/scripts/sbom/sbom/validate.py index efa4496ab9..99e224510a 100644 --- a/scripts/sbom/sbom/validate.py +++ b/scripts/sbom/sbom/validate.py @@ -11,6 +11,7 @@ import hashlib import json +import re import zipfile from pathlib import Path @@ -18,6 +19,30 @@ from . import document as document_module from . import wheelfile + +class ManifestError(Exception): + """A manifest is not shaped the way this tool writes them.""" + + +def _record_for(manifest: dict, filename: str) -> dict: + """One wheel's record, or a clear error rather than a KeyError traceback.""" + try: + records = manifest["wheels"] + except (KeyError, TypeError) as error: + raise ManifestError("manifest has no 'wheels' list") from error + for record in records: + if record.get("filename") == filename: + for key in ("sha256", "size", "sbom_in_wheel", "sidecars"): + if key not in record: + raise ManifestError(f"{filename}: manifest record has no {key!r}") + return record + return {} + + +_EVIDENCE_DIGEST = re.compile( + r"\(sha256:(?P[0-9a-f]{64})[^)]*\) packaged at (?P\S+)" +) + _REQUIRED_KEYS = ( "spdxVersion", "dataLicense", @@ -158,6 +183,51 @@ def verify_wheel( failures.extend(_check_record(wheel_path, wheel)) failures.extend(_check_licenses(wheel, spdx)) + failures.extend(_check_license_texts(wheel, spdx)) + return failures + + +def _check_license_texts(wheel: wheelfile.WheelInfo, spdx: dict) -> list[str]: + """Every text the document describes must be a text the wheel carries. + + The document states a digest for each licence it read and carries the + verbatim text of each LicenseRef. The wheel packages those same bytes under + `licenses/`. Nothing compared the two, so a packaged licence could be + swapped for another -- or emptied -- and every per-file digest still agreed, + because those describe the replacement. + """ + failures: list[str] = [] + by_name = {entry.name: entry.sha256 for entry in wheel.entries} + packaged = { + digest + for name, digest in by_name.items() + if name.startswith(f"{wheel.dist_info}/licenses/") + } + if not packaged: + return failures + + for package in spdx.get("packages", []): + for text in package.get("attributionTexts", []): + match = _EVIDENCE_DIGEST.search(text) + if not match: + continue + # The path matters, not just the bytes existing somewhere: two + # components can ship an identical text, so a swap inside one of them + # survives a check that only asks whether the digest is present. + where = match.group("where") + if by_name.get(where) != match.group("digest"): + failures.append( + f"{package['name']}: {where} does not hold the licence text " + f"this document records for it (sha256:{match.group('digest')})" + ) + + for extracted in spdx.get("hasExtractedLicensingInfos", []): + body = extracted.get("extractedText", "").encode("utf-8") + if hashlib.sha256(body).hexdigest() not in packaged: + failures.append( + f"{extracted['licenseId']}: the text it carries is not packaged " + "in this wheel" + ) return failures @@ -280,10 +350,8 @@ def _check_manifest( """The manifest binds a wheel's digest and names the sidecars beside it.""" failures: list[str] = [] manifest = json.loads(manifest_path.read_text(encoding="utf-8")) - record = next( - (item for item in manifest["wheels"] if item["filename"] == wheel.name), None - ) - if record is None: + record = _record_for(manifest, wheel.name) + if not record: return [f"{manifest_path.name} has no entry for {wheel.name}"] digest = hashlib.sha256(wheel.read_bytes()).hexdigest() @@ -315,13 +383,20 @@ def check_set(manifest_path: Path, wheel_dir: Path, evidence_dir: Path) -> list[ """Run :func:`check` over every wheel a merged manifest advertises.""" failures: list[str] = [] manifest = json.loads(manifest_path.read_text(encoding="utf-8")) - for record in manifest["wheels"]: + for entry in manifest.get("wheels") or []: + record = _record_for(manifest, entry.get("filename", "")) wheel = wheel_dir / record["filename"] if not wheel.is_file(): failures.append(f"{record['filename']} is missing from {wheel_dir}") continue failures.extend( f"{record['filename']}: {item}" - for item in check(wheel, manifest=manifest_path, evidence_dir=evidence_dir) + for item in check( + wheel, + build_evidence=evidence_dir + / record["sidecars"]["build_evidence"]["filename"], + manifest=manifest_path, + evidence_dir=evidence_dir, + ) ) return failures diff --git a/scripts/sbom/sbom/wheelfile.py b/scripts/sbom/sbom/wheelfile.py index dba048309d..127ce49ac3 100644 --- a/scripts/sbom/sbom/wheelfile.py +++ b/scripts/sbom/sbom/wheelfile.py @@ -144,12 +144,24 @@ def scan(wheel_path: Path, analyze_elf: bool = False) -> WheelInfo: """ with zipfile.ZipFile(wheel_path) as archive: names = archive.namelist() + # A zip may hold two members under one name. Reading by name gives the + # last of them, so the first is hashed by nothing, compared to RECORD by + # nothing, and reported as uncovered by nothing -- while which one a + # consumer extracts is up to their unzip. Refuse the archive. + repeated = sorted({item for item in names if names.count(item) > 1}) + if repeated: + raise WheelError( + f"more than one member is named {repeated}; a wheel member name " + "has to identify one file" + ) dist_info = _dist_info(names) entries: list[Entry] = [] for info in archive.infolist(): if info.is_dir(): continue - data = archive.read(info.filename) + # By handle, not by name: the name is what was just proven ambiguous. + with archive.open(info) as handle: + data = handle.read() soname = needed = machine = build_id = None if analyze_elf and elf.is_elf(data[:4]): dynamic = elf.read_dynamic(data) @@ -244,6 +256,11 @@ def read_record(wheel_path: Path, dist_info: str) -> dict[str, tuple[str, str]]: } +def _members(path: Path) -> list[zipfile.ZipInfo]: + with zipfile.ZipFile(path) as archive: + return [item for item in archive.infolist() if not item.is_dir()] + + def rewrite( wheel_path: Path, output_path: Path, @@ -258,6 +275,13 @@ def rewrite( digest cannot survive the step that writes it. """ record_name = f"{dist_info}/RECORD" + collisions = sorted( + set(additions) & {item.filename for item in _members(wheel_path)} + ) + if collisions: + raise WheelError( + f"these members already exist and would be added again: {collisions}" + ) if record_name in additions or record_name in replacements: raise WheelError("RECORD is regenerated here; do not pass it in") diff --git a/tests/python/sbom/test_gates.py b/tests/python/sbom/test_gates.py index ee21e1ac7a..45d2237140 100644 --- a/tests/python/sbom/test_gates.py +++ b/tests/python/sbom/test_gates.py @@ -13,6 +13,7 @@ import synth from sbom import build as build_module from sbom import licensing +from sbom import wheelfile from sbom import validate as validate_module DIST_INFO = synth.DIST_INFO @@ -407,6 +408,9 @@ def test_the_excluded_list_cannot_exempt_a_smuggled_file(built): """ with zipfile.ZipFile(built["wheel"]) as archive: payload = {item: archive.read(item) for item in archive.namelist()} + # write_wheel regenerates RECORD; keeping the old one would make a duplicate + # member, which is a different finding and now refused outright. + payload.pop(f"{DIST_INFO}/RECORD", None) payload["isaaccapture/_backdoor.py"] = b"import os\n" document = f"{DIST_INFO}/sboms/{WHEEL_NAME}.spdx.json" @@ -423,3 +427,24 @@ def test_the_excluded_list_cannot_exempt_a_smuggled_file(built): assert any("excluded-files list must be exactly" in item for item in failures) assert any("not covered by the SBOM" in item for item in failures) + + +def test_two_members_under_one_name_are_refused(built): + """Reading by name yields the last copy, so the first is hashed by nothing. + + Which one a consumer extracts is up to their unzip, so an archive carrying + both is not describable and must not be accepted. + """ + with zipfile.ZipFile(built["wheel"]) as archive: + names = [item for item in archive.namelist() if not item.endswith("/")] + payload = {item: archive.read(item) for item in names} + + smuggled = built["wheel"].with_name("duplicated.whl") + with zipfile.ZipFile(smuggled, "w", zipfile.ZIP_DEFLATED) as archive: + for name in names: + if name == "isaaccapture/__init__.py": + archive.writestr(name, b"import os\n") + archive.writestr(name, payload[name]) + + with pytest.raises(wheelfile.WheelError, match="more than one member"): + validate_module.check(smuggled) From fcf8e9410d34b12da4f5cbb7ee72738b121147a3 Mon Sep 17 00:00:00 2001 From: Andrei Aristarkhov Date: Wed, 30 Sep 2026 23:51:08 -0700 Subject: [PATCH 025/101] build(sbom): trace the project's own licence texts instead of asserting them The whitelist that stopped `.dist-info/` being a blanket exemption still exempted everything under `licenses/`, so the project's own texts were published as "distribution metadata written by the build" when they are copies of files this repository tracks -- `LICENSES/Apache-2.0.txt` in the wheel is byte for byte the one in the checkout. An origin was asserted for files that have a real one. They trace now. The texts this collector packages are unaffected: it attributes those as it adds them, which is why they already read as its own evidence rather than as metadata. Doing that exposed the path tiebreak preferring a dependency's source tree over this repository, which credited our BSL-1.0 pool text to an SDK that ships one under the same two trailing path segments. A member matching a file this project tracks is this project's copy; checked-in third-party code still carries its holder, because that branch looks the owner up. One attribution changed in the whole wheel, and it is that one. Signed-off-by: Andrei Aristarkhov --- scripts/sbom/sbom/inventory.py | 25 ++++++++++++++++++------- 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/scripts/sbom/sbom/inventory.py b/scripts/sbom/sbom/inventory.py index 4908d82613..c9a174ed62 100644 --- a/scripts/sbom/sbom/inventory.py +++ b/scripts/sbom/sbom/inventory.py @@ -496,6 +496,18 @@ def _resolve_by_path( artifacts[0], wheel_path, f"{source}; modified build output" ) + # This repository first. A member matching a file this project tracks is + # this project's copy; a dependency whose tree happens to end in the same + # two path segments is a coincidence -- our own BSL-1.0 pool text was + # being credited to an SDK that ships one under the same relative path. + # A third-party file checked in here still carries its component, because + # the repository branch looks its holder up. + repo_file = self.repo_files.path_suffix_match(wheel_path) + if repo_file: + same = self.repo_files.digest_of_display(repo_file) == digest + lead = f"{source}; {'identical to' if same else 'derived from'} " + return self._from_repo_file(wheel_path, source, repo_file, lead, same) + component_file = self.source_files.path_suffix_match(wheel_path) if component_file: key = component_file.split("/", 1)[0].removesuffix("-src") @@ -523,12 +535,6 @@ def _resolve_by_path( primary=key, ) - repo_file = self.repo_files.path_suffix_match(wheel_path) - if repo_file: - same = self.repo_files.digest_of_display(repo_file) == digest - lead = f"{source}; {'identical to' if same else 'derived from'} " - return self._from_repo_file(wheel_path, source, repo_file, lead, same) - build_file = self.build_files.path_suffix_match(wheel_path) if build_file: owner = self._staged_component_of(build_file) @@ -639,9 +645,14 @@ def _resolve_shared_by_sibling(attributions: dict[str, Attribution]) -> None: item.primary = key +# What a packaging backend writes, and SBOM documents other tools leave here. +# `licenses/` is deliberately absent: the project's own texts are copies of files +# this repository tracks, so they trace like any other copy, and the texts this +# collector adds are attributed as it adds them. Calling the directory metadata +# asserted an origin for files that have a real one. _DIST_INFO_WRITTEN = re.compile( r"^(METADATA|WHEEL|RECORD|INSTALLER|REQUESTED|entry_points\.txt|top_level\.txt" - r"|direct_url\.json|zip-safe|namespace_packages\.txt)$|^(licenses|sboms)/" + r"|direct_url\.json|zip-safe|namespace_packages\.txt)$|^sboms/" ) From 1ba82fe148b5b6acbeb69437a41956b08108bbe4 Mon Sep 17 00:00:00 2001 From: Andrei Aristarkhov Date: Thu, 1 Oct 2026 00:00:59 -0700 Subject: [PATCH 026/101] build(sbom): one entry per licence text, not one per component A LicenseRef identifies a text, so components carrying byte-identical terms -- two builds of one SDK, a vendor licence applied across products -- were minting an id each and carrying the whole text twice. Nothing about that is particular to the components that happen to collide here: entries are grouped by their text, and where several share one they share an id named after the text's digest, so it belongs to no single component and reads as shared. Every reference is rewritten to it. The document is 5.6% smaller for this wheel, which is where its weight actually was -- the texts are a fifth of it and the digests under a tenth. Signed-off-by: Andrei Aristarkhov --- scripts/sbom/sbom/document.py | 60 +++++++++++++++++++++++++--- tests/python/sbom/test_wheel_sbom.py | 40 +++++++++++++++++++ 2 files changed, 95 insertions(+), 5 deletions(-) diff --git a/scripts/sbom/sbom/document.py b/scripts/sbom/sbom/document.py index 82f29dbe85..6ad204857f 100644 --- a/scripts/sbom/sbom/document.py +++ b/scripts/sbom/sbom/document.py @@ -83,7 +83,9 @@ def _component_package(component: Component, roles: set[str], dist_info: str) -> return package -def _extracted_licenses(components: list[Component]) -> list[dict]: +def _extracted_licenses( + components: list[Component], +) -> tuple[list[dict], dict[str, str]]: """Texts for every LicenseRef the document uses, taken from the shipped file.""" entries: dict[str, dict] = {} for component in components: @@ -112,7 +114,46 @@ def _extracted_licenses(components: list[Component]) -> list[dict]: + "." ), } - return [entries[key] for key in sorted(entries)] + return _share_identical_texts([entries[key] for key in sorted(entries)]) + + +def _share_identical_texts(entries: list[dict]) -> tuple[list[dict], dict[str, str]]: + """One entry per distinct text, not one per component that ships it. + + Components can carry byte-identical terms -- two builds of one SDK, a licence + a vendor applies across products -- and a LicenseRef is an identifier for a + text, so minting one each duplicates the text in full. Where several share + one, they share one id, named after the text so it belongs to no single + component. + """ + by_text: dict[str, list[dict]] = {} + for entry in entries: + by_text.setdefault(entry["extractedText"], []).append(entry) + + shared: list[dict] = [] + renamed: dict[str, str] = {} + for text, group in by_text.items(): + if len(group) == 1: + shared.append(group[0]) + continue + digest = hashlib.sha256(text.encode("utf-8")).hexdigest()[:12] + canonical = f"LicenseRef-shared-text-{digest}" + for entry in group: + renamed[entry["licenseId"]] = canonical + shared.append( + { + **group[0], + "licenseId": canonical, + "name": "License terms shipped with " + + ", ".join( + sorted( + item["name"].removeprefix("License terms shipped with ") + for item in group + ) + ), + } + ) + return sorted(shared, key=lambda item: item["licenseId"]), renamed _EXTRA_MARKER = re.compile(r"\bextra\s*==") @@ -384,6 +425,17 @@ def build_document( # rebuild be compared against a published one byte for byte. stamp = stamped_now() + extracted, renamed = _extracted_licenses( + list(inventory.components_present.values()) + ) + if renamed: + # Every reference to a text that is now shared points at the shared id; + # leaving one behind would use a LicenseRef the document stops defining. + for package in packages: + for key in ("licenseConcluded", "licenseDeclared"): + for old_id, new_id in renamed.items(): + package[key] = package[key].replace(old_id, new_id) + return { "spdxVersion": SPDX_VERSION, "dataLicense": "CC0-1.0", @@ -408,7 +460,5 @@ def build_document( "packages": packages, "files": files, "relationships": relationships, - "hasExtractedLicensingInfos": _extracted_licenses( - list(inventory.components_present.values()) - ), + "hasExtractedLicensingInfos": extracted, } diff --git a/tests/python/sbom/test_wheel_sbom.py b/tests/python/sbom/test_wheel_sbom.py index 385ea74831..c35d36609c 100644 --- a/tests/python/sbom/test_wheel_sbom.py +++ b/tests/python/sbom/test_wheel_sbom.py @@ -6,6 +6,7 @@ from __future__ import annotations import json +import re import zipfile import synth @@ -253,3 +254,42 @@ def test_source_date_epoch_normalizes_timestamps_the_input_wheel_carried( with zf.ZipFile(workspace.wheel) as archive: stamps = {info.date_time for info in archive.infolist()} assert len(stamps) == 1, f"members carry differing timestamps: {stamps}" + + +def test_components_sharing_a_licence_text_share_one_entry(workspace, license_data): + """A LicenseRef identifies a text, so one text is one entry. + + Two components can carry byte-identical terms -- two builds of one SDK, a + vendor licence applied across products -- and minting an id each duplicates + the whole text in the document. + """ + from sbom import licensing + + licensing.load_corpus(license_data) + for key in ("alpha", "beta"): + root = workspace.build / "_deps" / f"{key}-src" + root.mkdir(parents=True, exist_ok=True) + (root / "LICENSE").write_text(synth.PROPRIETARY_LICENSE) + synth.track(workspace.root) + + payload = synth.wheel_payload(workspace) + for key in ("alpha", "beta"): + payload[f"isaaccapture/{key}_notice.txt"] = synth.PROPRIETARY_LICENSE.encode() + synth.write_wheel(workspace.wheel, payload) + build_module.build( + workspace.root, workspace.build, workspace.wheel, workspace.root / "sbom" + ) + + with zipfile.ZipFile(workspace.wheel) as archive: + spdx = json.loads(archive.read(f"{DIST_INFO}/sboms/{WHEEL_NAME}.spdx.json")) + bodies = [item["extractedText"] for item in spdx["hasExtractedLicensingInfos"]] + + assert len(bodies) == len(set(bodies)), "a text is carried more than once" + used = { + ref + for package in spdx["packages"] + for field in ("licenseConcluded", "licenseDeclared") + for ref in re.findall(r"LicenseRef-[\w.-]+", package.get(field, "")) + } + defined = {item["licenseId"] for item in spdx["hasExtractedLicensingInfos"]} + assert used <= defined, f"undefined after sharing: {sorted(used - defined)}" From 9521980ac0c99a811ef32363f661dd69c598d57b Mon Sep 17 00:00:00 2001 From: Andrei Aristarkhov Date: Thu, 1 Oct 2026 00:20:17 -0700 Subject: [PATCH 027/101] build(sbom): hold the document to the evidence, not to itself A seventh round, adversarial again. It got five artifacts past the gates; all five are refused now, and the attacks it failed to land are still refused. Verification checked the document's file list and digests and nothing else the document asserts. Every package-level statement -- which components are here, under what terms, which text belongs to which -- could be deleted or rewritten with `check` still green: 35 packaged licence texts removed, or a whole component dropped, and the wheel still claimed to redistribute them. The build gate asks whether a text existed to read; verification now asks whether the wheel in hand still carries it. The licence-text check read the expected digest out of the string it was checking, so a rewrite consistent with itself was invisible -- while the sidecar holding the true digest was open in the same call. Where the evidence is supplied it is what the document is compared against. Without it a self- consistent forgery remains undetectable, which is the trust model: this describes a wheel, a signature attests one. LicenseRef ids were minted from an archive's basename in one place and a soname in another, while the package ids beside them use the component key. Two archives in different directories sharing a name minted one id for two different texts, and the last writer won -- handing one vendor's SDK another's terms. Ids come from the key everywhere, and minting one twice for different text is an error rather than an overwrite. Also: the id rewrite introduced with shared texts was a substring replace, so a ref that is a prefix of another was mangled into one nothing defines; a member named as a directory could carry bytes nothing accounted for; a manifest could advertise one filename twice, which the producer refuses and the verifier resolved to the first match; a named-but-absent sidecar tracebacked; and the grant-or-pool rule had a fourth spelling. Signed-off-by: Andrei Aristarkhov --- scripts/sbom/sbom/discovery.py | 5 +- scripts/sbom/sbom/document.py | 27 ++++++- scripts/sbom/sbom/evidence.py | 9 ++- scripts/sbom/sbom/inventory.py | 11 ++- scripts/sbom/sbom/validate.py | 111 ++++++++++++++++++++++----- scripts/sbom/sbom/wheelfile.py | 9 +++ tests/python/sbom/test_wheel_sbom.py | 10 +-- 7 files changed, 150 insertions(+), 32 deletions(-) diff --git a/scripts/sbom/sbom/discovery.py b/scripts/sbom/sbom/discovery.py index 59a42dfd23..25850e5967 100644 --- a/scripts/sbom/sbom/discovery.py +++ b/scripts/sbom/sbom/discovery.py @@ -798,7 +798,10 @@ def system_component(soname: str, origin: dict) -> Component: evidence.append( licensing._evidence(key, "build-host", license_path, text, "grant") # noqa: SLF001 ) - concluded, declared = licensing.expression(evidence, soname) + # `key`, not `soname`: the key is what identifies this component everywhere + # else, and two components reduced to the same label would mint one LicenseRef + # for two different texts. + concluded, declared = licensing.expression(evidence, key) package = origin.get("package") details = [ diff --git a/scripts/sbom/sbom/document.py b/scripts/sbom/sbom/document.py index 6ad204857f..7ad781b230 100644 --- a/scripts/sbom/sbom/document.py +++ b/scripts/sbom/sbom/document.py @@ -92,10 +92,19 @@ def _extracted_licenses( refs = licensing.license_refs( component.license_concluded, component.license_declared ) - grants = [item for item in component.evidence if item.kind == "grant"] + # Same predicate as the build gate, the verifier and the report: a REUSE + # pool is a real text. Written four ways, it was a rule in four places. + grants = [item for item in component.evidence if item.kind in ("grant", "pool")] if not refs or not grants: continue for ref in refs: + if ref in entries and entries[ref]["extractedText"] != "\n\n".join( + item.text for item in grants + ): + raise ValueError( + f"{ref} was minted for two components with different terms; " + "a LicenseRef identifies one text" + ) entries[ref] = { "licenseId": ref, "name": f"License terms shipped with {component.name}", @@ -117,6 +126,19 @@ def _extracted_licenses( return _share_identical_texts([entries[key] for key in sorted(entries)]) +def _rename_refs(expression: str, renamed: dict[str, str]) -> str: + """Substitute whole identifiers, never substrings. + + `LicenseRef-alpha` is a prefix of `LicenseRef-alpha-extra`, so a plain + replace turns the longer id into one nothing defines and orphans its entry. + """ + for old_id, new_id in renamed.items(): + expression = re.sub( + rf"(? tuple[list[dict], dict[str, str]]: """One entry per distinct text, not one per component that ships it. @@ -433,8 +455,7 @@ def build_document( # leaving one behind would use a LicenseRef the document stops defining. for package in packages: for key in ("licenseConcluded", "licenseDeclared"): - for old_id, new_id in renamed.items(): - package[key] = package[key].replace(old_id, new_id) + package[key] = _rename_refs(package[key], renamed) return { "spdxVersion": SPDX_VERSION, diff --git a/scripts/sbom/sbom/evidence.py b/scripts/sbom/sbom/evidence.py index ebcc9cd5dc..81aeefb77f 100644 --- a/scripts/sbom/sbom/evidence.py +++ b/scripts/sbom/sbom/evidence.py @@ -128,7 +128,9 @@ def _archive_components( ) for member, data in record.get("license_members", []) ] - concluded, declared = licensing.expression(evidence, name) + # `display`, not `name`: the name is the archive's basename, so two + # archives in different directories would share a LicenseRef. + concluded, declared = licensing.expression(evidence, display) source_url = _fetched_from(repo_root / display) held = ( @@ -366,6 +368,11 @@ def document(discovery: Discovery, wheel_name: str, inventory) -> dict: }, "build": { "build_dir": str(discovery.build_dir), + # Named once: a member's origin reads "staged" rather than repeating + # this prefix and the member's own path for every file staged here. + "staged_root": ( + str(discovery.staged_root) if discovery.staged_root else None + ), "config": discovery.config, "arch": arch, "platform": platform.platform(), diff --git a/scripts/sbom/sbom/inventory.py b/scripts/sbom/sbom/inventory.py index c9a174ed62..92b0e51775 100644 --- a/scripts/sbom/sbom/inventory.py +++ b/scripts/sbom/sbom/inventory.py @@ -174,7 +174,16 @@ def resolve( # -- and saying the member is a copy of something it does not hash to # would describe a file the wheel does not contain. patched = candidate_digest != digest - source = f"staged at {probe.relative_to(self.build_dir).as_posix()}" + relative = probe.relative_to(self.build_dir).as_posix() + # The staged path is the staging root plus the member's own path + # unless the build renamed the file on the way in. Printing it + # whenever it is derivable repeats the member path once per member + # and says nothing; where it is not, it is the interesting part. + source = ( + "staged" + if relative.endswith(f"/{wheel_path}") + else f"staged at {relative}" + ) found = self._trace( wheel_path, candidate_digest, digest, source, soname, build_id diff --git a/scripts/sbom/sbom/validate.py b/scripts/sbom/sbom/validate.py index 99e224510a..6d17064d77 100644 --- a/scripts/sbom/sbom/validate.py +++ b/scripts/sbom/sbom/validate.py @@ -24,12 +24,24 @@ class ManifestError(Exception): """A manifest is not shaped the way this tool writes them.""" +def _sidecar(evidence_dir: Path, record: dict, kind: str) -> Path | None: + """A named sidecar, or None so the missing-file failure is reported as one.""" + named = record.get("sidecars", {}).get(kind, {}).get("filename") + if not named: + return None + path = evidence_dir / named + return path if path.is_file() else None + + def _record_for(manifest: dict, filename: str) -> dict: """One wheel's record, or a clear error rather than a KeyError traceback.""" try: records = manifest["wheels"] except (KeyError, TypeError) as error: raise ManifestError("manifest has no 'wheels' list") from error + seen = [item for item in records if item.get("filename") == filename] + if len(seen) > 1: + raise ManifestError(f"{filename} is advertised {len(seen)} times") for record in records: if record.get("filename") == filename: for key in ("sha256", "size", "sbom_in_wheel", "sidecars"): @@ -221,13 +233,6 @@ def _check_license_texts(wheel: wheelfile.WheelInfo, spdx: dict) -> list[str]: f"this document records for it (sha256:{match.group('digest')})" ) - for extracted in spdx.get("hasExtractedLicensingInfos", []): - body = extracted.get("extractedText", "").encode("utf-8") - if hashlib.sha256(body).hexdigest() not in packaged: - failures.append( - f"{extracted['licenseId']}: the text it carries is not packaged " - "in this wheel" - ) return failures @@ -265,6 +270,12 @@ def _check_attribution(evidence_doc: dict, wheel: wheelfile.WheelInfo) -> list[s failures: list[str] = [] attributed = {item["path"] for item in evidence_doc.get("attributions", [])} + packaged = { + entry.sha256 + for entry in wheel.entries + if entry.name.startswith(f"{wheel.dist_info}/licenses/") + } + recorded: set[str] = set() for entry in wheel.entries: if entry.name not in attributed: @@ -275,18 +286,79 @@ def _check_attribution(evidence_doc: dict, wheel: wheelfile.WheelInfo) -> list[s for key, component in evidence_doc.get("components", {}).items(): if not component.get("in_this_wheel"): continue + recorded.update( + item["sha256"] + for item in component.get("evidence", []) + if item["kind"] in ("grant", "pool") + ) # The same test the build gate applies: a REUSE pool is a real text even # though it names no single expression, so publishing on one and then # failing verification for want of a grant would contradict the gate. - if not any( - item["kind"] in ("grant", "pool") for item in component.get("evidence", []) - ): + texts = [ + item + for item in component.get("evidence", []) + if item["kind"] in ("grant", "pool") + ] + if not texts: failures.append( f"{key} is redistributed but no license text was found for it" ) + continue + # Found is not shipped. The build gate asks whether a text existed to + # read; this asks whether the wheel in hand still carries it. Without + # the second, every statement the document makes about a component -- + # that it is here, under these terms, with this text -- could be deleted + # along with the text and nothing would notice. + if not any(item["sha256"] in packaged for item in texts): + failures.append( + f"{key} is redistributed but the license text recorded for it is " + "not packaged in this wheel" + ) + failures.extend(_document_agrees_with_evidence(wheel, recorded)) + return failures + + +def _document_agrees_with_evidence( + wheel: wheelfile.WheelInfo, recorded: set[str] +) -> list[str]: + """The texts the document names must be the ones the build actually read. + + `_check_license_texts` reads the expected digest out of the document it is + checking, so a rewrite consistent with itself is invisible to it. The build + evidence was produced before the wheel was published and states the digests + independently, so where it is supplied it is the thing to compare against. + """ + import zipfile as _zipfile + + with _zipfile.ZipFile(wheel.path) as archive: + name = wheelfile.sbom_member(wheel.dist_info, wheel.path.name) + if name not in archive.namelist(): + return [] + spdx = json.loads(archive.read(name)) + + failures = [] + for package in spdx.get("packages", []): + for text in package.get("attributionTexts", []): + match = _EVIDENCE_DIGEST.search(text) + if match and match.group("digest") not in recorded: + failures.append( + f"{package['name']}: the document names a licence text " + f"(sha256:{match.group('digest')}) the build never read" + ) return failures +def _refs_in(value) -> set[str]: + """LicenseRef identifiers in a field that may hold an expression or a list.""" + if value is None: + return set() + items = value if isinstance(value, list) else [value] + found: set[str] = set() + for item in items: + found.update(licensing.license_refs(str(item))) + return found + + def _check_licenses(wheel: wheelfile.WheelInfo, spdx: dict) -> list[str]: failures: list[str] = [] names = {item.name for item in wheel.entries} @@ -294,18 +366,16 @@ def _check_licenses(wheel: wheelfile.WheelInfo, spdx: dict) -> list[str]: if notices not in names: failures.append(f"{notices} is not packaged") + # Every field that can name one, through one parser. Two of these were not + # looked at, and packages were tokenised by hand while files went through the + # grammar -- two readings of one notation, disagreeing at the edges. declared_refs = set() for item in spdx.get("files", []): - declared_refs.update( - licensing.license_refs(str(item.get("licenseConcluded", ""))) - ) + for field in ("licenseConcluded", "licenseInfoInFiles"): + declared_refs.update(_refs_in(item.get(field))) for package in spdx["packages"]: - for field in ("licenseConcluded", "licenseDeclared"): - for token in ( - str(package.get(field, "")).replace("(", " ").replace(")", " ").split() - ): - if token.startswith("LicenseRef-"): - declared_refs.add(token) + for field in ("licenseConcluded", "licenseDeclared", "licenseInfoFromFiles"): + declared_refs.update(_refs_in(package.get(field))) extracted = { item["licenseId"] for item in spdx.get("hasExtractedLicensingInfos", []) } @@ -393,8 +463,7 @@ def check_set(manifest_path: Path, wheel_dir: Path, evidence_dir: Path) -> list[ f"{record['filename']}: {item}" for item in check( wheel, - build_evidence=evidence_dir - / record["sidecars"]["build_evidence"]["filename"], + build_evidence=_sidecar(evidence_dir, record, "build_evidence"), manifest=manifest_path, evidence_dir=evidence_dir, ) diff --git a/scripts/sbom/sbom/wheelfile.py b/scripts/sbom/sbom/wheelfile.py index 127ce49ac3..c4d866e940 100644 --- a/scripts/sbom/sbom/wheelfile.py +++ b/scripts/sbom/sbom/wheelfile.py @@ -156,6 +156,15 @@ def scan(wheel_path: Path, analyze_elf: bool = False) -> WheelInfo: ) dist_info = _dist_info(names) entries: list[Entry] = [] + carrying = [ + item.filename + for item in archive.infolist() + if item.is_dir() and item.file_size + ] + if carrying: + raise WheelError( + f"these members are named as directories but carry bytes: {carrying}" + ) for info in archive.infolist(): if info.is_dir(): continue diff --git a/tests/python/sbom/test_wheel_sbom.py b/tests/python/sbom/test_wheel_sbom.py index c35d36609c..884cf5da9a 100644 --- a/tests/python/sbom/test_wheel_sbom.py +++ b/tests/python/sbom/test_wheel_sbom.py @@ -145,11 +145,11 @@ def test_an_unidentified_grant_travels_verbatim_in_the_document(built): spdx = _embedded(built["wheel"]) extracted = {item["licenseId"]: item for item in spdx["hasExtractedLicensingInfos"]} - assert "LicenseRef-demosdk-1.2.0-linux-amd64" in extracted - assert ( - "DEMO CORP" - in extracted["LicenseRef-demosdk-1.2.0-linux-amd64"]["extractedText"] - ) + # The id is minted from the component's key -- its path, here -- not from a + # basename two archives in different directories would share. + ref = next(item for item in extracted if "demosdk" in item) + assert "deps" in ref and "vendor" in ref, ref + assert "DEMO CORP" in extracted[ref]["extractedText"] def test_reuse_tags_on_shipped_files_are_reported_per_file(built): From 9c2766f477298dadd81cef5359f541c3f2101172 Mon Sep 17 00:00:00 2001 From: Andrei Aristarkhov Date: Thu, 1 Oct 2026 00:29:07 -0700 Subject: [PATCH 028/101] build(sbom): give the distribution's own package its copyright notices The wheel package declared a supplier, a licence and no copyright, beside files that each state one -- the package saying less about itself than anything in it. It carries the notices its own files state now, which is the same derivation the licence roll-up beside it already uses. A file attributed to a component is excluded: its notice belongs to that component's package. The companion wheel ships no file bearing one and so still asserts nothing, rather than being given a notice no file states. Also corrects the coverage-threshold comment, which put a genuine match at 30% or above; a Debian copyright file naming several licences around the one matched sits at a quarter, and is the reason the threshold is where it is. Signed-off-by: Andrei Aristarkhov --- scripts/sbom/sbom/document.py | 11 ++++++++++- scripts/sbom/sbom/licensing.py | 6 +++--- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/scripts/sbom/sbom/document.py b/scripts/sbom/sbom/document.py index 7ad781b230..4e56ea591c 100644 --- a/scripts/sbom/sbom/document.py +++ b/scripts/sbom/sbom/document.py @@ -229,6 +229,10 @@ def build_document( # What the files themselves state, for the package-level roll-up below. stated_in_files: set[str] = set() + # Notices stated by files this distribution's own, for the package below. A + # file attributed to a component carries that component's notice, which + # belongs to that component's package and not to this one. + stated_copyright: set[str] = set() for index, entry in enumerate(sorted(wheel.entries, key=lambda item: item.name)): if entry.name in excluded_files: continue @@ -255,6 +259,8 @@ def build_document( record["licenseConcluded"] = attribution.spdx_tag record["licenseInfoInFiles"] = [attribution.spdx_tag] stated_in_files.add(attribution.spdx_tag) + if attribution and attribution.copyright_text and not attribution.primary: + stated_copyright.add(attribution.copyright_text) comment = [] if attribution: @@ -285,7 +291,10 @@ def build_document( "licenseConcluded": project["license"], "licenseDeclared": project["license"], "licenseInfoFromFiles": sorted(stated_in_files) or [NOASSERTION], - "copyrightText": NOASSERTION, + # The notices its own files carry. Declaring none beside a supplier, a + # licence and files that each state one left the package saying less + # about itself than anything in it. + "copyrightText": "\n".join(sorted(stated_copyright)) or NOASSERTION, "packageVerificationCode": { "packageVerificationCodeValue": code, "packageVerificationCodeExcludedFiles": [ diff --git a/scripts/sbom/sbom/licensing.py b/scripts/sbom/sbom/licensing.py index d1919de9ae..1498f6fc65 100644 --- a/scripts/sbom/sbom/licensing.py +++ b/scripts/sbom/sbom/licensing.py @@ -42,9 +42,9 @@ _CONTAINMENT_THRESHOLD = 0.90 # ...and has to account for a real share of the file it was found in. Large # agreements (the CUDA EULA, for one) reproduce whole OSS licenses in an -# appendix; those reach full containment at about 3% coverage, while a genuine -# match -- even a Debian copyright file wrapped in packaging boilerplate -- sits -# at 30% or above. +# appendix; those reach full containment at a few percent coverage, while a +# genuine match sits well above it -- a Debian copyright file naming several +# licenses around the one matched is the low end, at a quarter. _COVERAGE_THRESHOLD = 0.20 # Two licenses are one family when either text is nearly inside the other. _FAMILY_THRESHOLD = 0.90 From 1db065e0d1fbb81d8a04b31a4688d4c428c0b655 Mon Sep 17 00:00:00 2001 From: Andrei Aristarkhov Date: Thu, 1 Oct 2026 00:37:41 -0700 Subject: [PATCH 029/101] build(sbom): fold a copyright notice another already covers Files written in different years state one holder over different year ranges, so the distribution's own package listed that holder three times over, differing only in the years. Merging the ranges would be the obvious fix and the wrong one: a union of 2021 and 2023 claims 2022, which no file claims. A notice is dropped only when its holder matches another's and that other's years already contain its own, so what is published is always a line some file states. Disjoint ranges both survive, and different holders never merge. Applies wherever notices are gathered, so a component assembled from several vendored files reads the same way. Signed-off-by: Andrei Aristarkhov --- scripts/sbom/sbom/discovery.py | 4 +++- scripts/sbom/sbom/document.py | 3 ++- scripts/sbom/sbom/licensing.py | 31 +++++++++++++++++++++++++++++ tests/python/sbom/test_licensing.py | 28 ++++++++++++++++++++++++++ 4 files changed, 64 insertions(+), 2 deletions(-) diff --git a/scripts/sbom/sbom/discovery.py b/scripts/sbom/sbom/discovery.py index 25850e5967..95af5419a2 100644 --- a/scripts/sbom/sbom/discovery.py +++ b/scripts/sbom/sbom/discovery.py @@ -576,7 +576,9 @@ def discover_vendored( ), license_concluded=concluded, license_declared=declared, - copyright_text="\n".join(sorted(notices.get(key, {names[key]}))), + copyright_text="\n".join( + licensing.fold_notices(notices.get(key, {names[key]})) + ), evidence=tuple(evidence), ) return components, grouped diff --git a/scripts/sbom/sbom/document.py b/scripts/sbom/sbom/document.py index 4e56ea591c..8f4afe7dbe 100644 --- a/scripts/sbom/sbom/document.py +++ b/scripts/sbom/sbom/document.py @@ -294,7 +294,8 @@ def build_document( # The notices its own files carry. Declaring none beside a supplier, a # licence and files that each state one left the package saying less # about itself than anything in it. - "copyrightText": "\n".join(sorted(stated_copyright)) or NOASSERTION, + "copyrightText": "\n".join(licensing.fold_notices(stated_copyright)) + or NOASSERTION, "packageVerificationCode": { "packageVerificationCodeValue": code, "packageVerificationCodeExcludedFiles": [ diff --git a/scripts/sbom/sbom/licensing.py b/scripts/sbom/sbom/licensing.py index 1498f6fc65..c39dde9cf6 100644 --- a/scripts/sbom/sbom/licensing.py +++ b/scripts/sbom/sbom/licensing.py @@ -492,6 +492,37 @@ def read_spdx_tag(text: str) -> str | None: return match.group("expression").strip() if match else None +_NOTICE_YEARS = re.compile(r"\b(\d{4})(?:\s*[-\u2013]\s*(\d{4}))?\b") + + +def fold_notices(notices) -> list[str]: + """Drop a notice another already covers. + + Files written in different years state one holder over different year ranges, + and listing every one names that holder several times over. Merging the + ranges instead would claim a year no file claims, so only a notice whose + holder matches another's and whose years that other already contains is + dropped -- it says nothing the one kept does not. + """ + parsed = [] + for notice in notices: + years: set[int] = set() + for start, end in _NOTICE_YEARS.findall(notice): + years.update(range(int(start), int(end or start) + 1)) + holder = re.sub(r"[^A-Za-z0-9]+", " ", _NOTICE_YEARS.sub("", notice)) + parsed.append((holder.strip().lower(), frozenset(years), notice)) + + return sorted( + { + notice + for holder, years, notice in parsed + if not any( + other == holder and years < covered for other, covered, _ in parsed + ) + } + ) + + def read_copyright(text: str) -> str | None: """The notice a file states about itself, beside its REUSE tag.""" match = _FILE_COPYRIGHT.search(text) diff --git a/tests/python/sbom/test_licensing.py b/tests/python/sbom/test_licensing.py index 4b0caf452d..407218dea7 100644 --- a/tests/python/sbom/test_licensing.py +++ b/tests/python/sbom/test_licensing.py @@ -249,3 +249,31 @@ def test_a_real_license_section_is_still_found(tmp_path): evidence = licensing.discover_in_tree("thing", tmp_path, "thing-src") assert [item.origin for item in evidence] == ["readme-section"] + + +def test_a_notice_another_already_covers_is_dropped(): + """One holder stated over several year ranges is one holder.""" + folded = licensing.fold_notices( + [ + "Copyright (c) 2025 Acme Ltd.", + "Copyright (c) 2025-2026 Acme Ltd.", + "Copyright (c) 2026 Acme Ltd.", + ] + ) + + assert folded == ["Copyright (c) 2025-2026 Acme Ltd."] + + +def test_folding_never_claims_a_year_no_file_claims(): + """Merging ranges would assert 2022; neither notice does.""" + assert licensing.fold_notices( + ["Copyright (c) 2021 Acme Ltd.", "Copyright (c) 2023 Acme Ltd."] + ) == ["Copyright (c) 2021 Acme Ltd.", "Copyright (c) 2023 Acme Ltd."] + + +def test_folding_does_not_merge_different_holders(): + folded = licensing.fold_notices( + ["Copyright (c) 2026 Acme Ltd.", "Copyright (c) 2026 Other Corp."] + ) + + assert len(folded) == 2 From e1c1d867404ad38af66bc805fdd83af906bb6eda Mon Sep 17 00:00:00 2001 From: Andrei Aristarkhov Date: Thu, 1 Oct 2026 00:42:44 -0700 Subject: [PATCH 030/101] build(sbom): read the supplier from the project, not from this file The distribution's supplier was a constant in the collector naming an organisation, so a fork, a rename or a transfer would have left it stating the wrong party with nothing to notice. The project already declares who it is in its own packaging metadata, which the vendored-component scan has been reading all along to decide which copyright holders are not its own. Those names are kept as written now and lowercased where the comparison needs it, rather than being folded on the way in and unavailable for anything else. Several authors name no single supplier and none names nobody, so both give NOASSERTION: SPDX takes one, and choosing would be this collector speaking for a project rather than reading it. Signed-off-by: Andrei Aristarkhov --- scripts/sbom/sbom/build.py | 11 ++++++++-- scripts/sbom/sbom/discovery.py | 21 +++++++++++++++--- tests/python/sbom/test_discovery.py | 33 +++++++++++++++++++++++++++++ 3 files changed, 60 insertions(+), 5 deletions(-) diff --git a/scripts/sbom/sbom/build.py b/scripts/sbom/sbom/build.py index 1e34848a37..10b257d135 100644 --- a/scripts/sbom/sbom/build.py +++ b/scripts/sbom/sbom/build.py @@ -20,11 +20,14 @@ from . import inventory as inventory_module from . import report as report_module from . import wheelfile +from .discovery import project_supplier from .inventory import Attribution from .wheelfile import Entry +# Supplier is absent on purpose: it is read from the project's own packaging +# metadata at build time, so a fork or a rename cannot leave this file naming +# somebody else's organisation with nothing to catch it. PROJECT = { - "supplier": "Organization: NVIDIA CORPORATION", "homepage": "https://github.com/NVIDIA/IsaacCapture", "license": "Apache-2.0", "document_namespace": "https://github.com/NVIDIA/IsaacCapture/spdx", @@ -169,7 +172,11 @@ def build( evidence_doc["system_libraries"] = resolver.system_libraries spdx = document_module.build_document( - PROJECT, evidence_doc, projected_wheel, inventory, excluded + {**PROJECT, "supplier": project_supplier(repo_root)}, + evidence_doc, + projected_wheel, + inventory, + excluded, ) spdx_bytes = json.dumps(spdx, indent=2, sort_keys=True).encode("utf-8") + b"\n" additions[sbom_name] = spdx_bytes diff --git a/scripts/sbom/sbom/discovery.py b/scripts/sbom/sbom/discovery.py index 95af5419a2..f803127ac4 100644 --- a/scripts/sbom/sbom/discovery.py +++ b/scripts/sbom/sbom/discovery.py @@ -484,7 +484,11 @@ def owners_under(self, directory: Path) -> set[str]: def project_authors(repo_root: Path) -> set[str]: - """Who the project says it is, read from its own packaging metadata.""" + """Who the project says it is, read from its own packaging metadata. + + As written, not folded: the same names answer "is this holder ours" and + "who supplied this distribution", and only the first wants them lowercased. + """ pyproject = repo_root / "pyproject.toml" if not pyproject.is_file(): return set() @@ -493,12 +497,23 @@ def project_authors(repo_root: Path) -> set[str]: except (OSError, tomllib.TOMLDecodeError): return set() return { - str(author["name"]).lower() + str(author["name"]) for author in data.get("project", {}).get("authors", []) if author.get("name") } +def project_supplier(repo_root: Path) -> str: + """Who supplied this distribution, as its own packaging metadata states it. + + SPDX takes one supplier, so several authors name nobody in particular and + none names nobody at all; either way asserting one would be this collector + speaking for a project rather than reading it. + """ + authors = project_authors(repo_root) + return f"Organization: {authors.pop()}" if len(authors) == 1 else "NOASSERTION" + + def _holder_of(path: Path) -> tuple[str, str | None] | None: """The copyright holder and licence a file states about itself.""" try: @@ -541,7 +556,7 @@ def discover_vendored( if stated is None: continue holder, expression = stated - if any(author in holder.lower() for author in authors): + if any(author.lower() in holder.lower() for author in authors): continue key = f"vendored:{licensing.spdx_safe(holder).lower()}" grouped.setdefault(key, []).append(path) diff --git a/tests/python/sbom/test_discovery.py b/tests/python/sbom/test_discovery.py index cc399d0969..a2b46b9a5f 100644 --- a/tests/python/sbom/test_discovery.py +++ b/tests/python/sbom/test_discovery.py @@ -268,3 +268,36 @@ def test_a_build_tree_with_no_configuration_says_so(workspace): with pytest.raises(evidence.EvidenceError, match="CMAKE_BUILD_TYPE"): evidence.discover(workspace.root, workspace.build) + + +def test_the_supplier_is_read_from_the_project_not_written_here(workspace): + """A fork or a rename must not leave the document naming the wrong party.""" + from sbom import discovery + + assert discovery.project_supplier(workspace.root) == "Organization: NVIDIA" + + pyproject = workspace.root / "pyproject.toml" + pyproject.write_text( + pyproject.read_text().replace( + '{ name = "NVIDIA" }', '{ name = "Someone Else" }' + ), + encoding="utf-8", + ) + + assert discovery.project_supplier(workspace.root) == "Organization: Someone Else" + + +def test_a_project_naming_several_authors_names_no_single_supplier(workspace): + """SPDX takes one; picking would be this tool speaking for the project.""" + from sbom import discovery + + pyproject = workspace.root / "pyproject.toml" + pyproject.write_text( + pyproject.read_text().replace( + '{ name = "NVIDIA" }', + '{ name = "NVIDIA" }, { name = "Other Party" }', + ), + encoding="utf-8", + ) + + assert discovery.project_supplier(workspace.root) == "NOASSERTION" From ccaf5662dd335d62014c02e2d6a3a126ae7a27a7 Mon Sep 17 00:00:00 2001 From: Andrei Aristarkhov Date: Thu, 1 Oct 2026 00:52:23 -0700 Subject: [PATCH 031/101] build(sbom): read the licence texts the collector itself packages Three questions about licence files, with three different answers. Ambiguous authorship now stops the build. Choosing among several authors, or publishing a release that names none, is for whoever is publishing it; the collector asked the question and answered it quietly. A licence file can state its own terms, and one here does -- OpenXR's COPYING.adoc is CC-BY-4.0 prose about which licences the project uses. These were the only members never read for a tag or a notice, because the collector packages them and so nothing scanned them the way it scans what it describes. Where such a file states its licence, that is now what it says; where it does not, it stays unasserted. The text of a licence is not itself under that licence and nothing here invents an authorship for it. The notice inside a component's own licence file is that component's, and for MIT and the BSDs it is a line of the same file -- shipped in the wheel, named nowhere a reader could act on. Components carry it now. Which is only true of a grant. A pool is a directory of reference texts, and the notice in one belongs to whoever wrote that licence: reading OpenXR's pool credited OpenXR to the author of WTFPL, and the distribution's own roll-up then credited itself to him too. A template names nobody and is not a notice. Signed-off-by: Andrei Aristarkhov --- scripts/sbom/sbom/build.py | 7 +++++ scripts/sbom/sbom/cli.py | 1 + scripts/sbom/sbom/discovery.py | 40 +++++++++++++++++++++++++---- scripts/sbom/sbom/document.py | 10 +++++++- scripts/sbom/sbom/evidence.py | 2 ++ scripts/sbom/sbom/inventory.py | 17 +----------- scripts/sbom/sbom/licensing.py | 37 ++++++++++++++++++++++++++ tests/python/sbom/test_discovery.py | 7 ++--- tests/python/sbom/test_licensing.py | 10 ++++++++ 9 files changed, 106 insertions(+), 25 deletions(-) diff --git a/scripts/sbom/sbom/build.py b/scripts/sbom/sbom/build.py index 10b257d135..d3ddf6f0ff 100644 --- a/scripts/sbom/sbom/build.py +++ b/scripts/sbom/sbom/build.py @@ -152,12 +152,19 @@ def build( ] projected_wheel = replace(wheel, entries=tuple(projected)) for name in [*additions, *replacements]: + body = additions.get(name, b"").decode("utf-8", "replace") + # A licence text can state its own terms and carry a notice, and these + # were the only members never read for either -- the collector packaged + # them, so nothing scanned them the way it scans what it is describing. inventory.attributions[name] = Attribution( path=name, origin="metadata", + exact=True, detail="license evidence packaged by the SBOM collector" if name in additions else "distribution metadata, updated to declare the packaged license files", + spdx_tag=licensing.normalized_tag(body) if name in additions else None, + copyright_text=licensing.read_notice(body) if name in additions else None, ) # The document is added below, after it has been rendered; record it now so # the evidence accounts for every member of the finished wheel. diff --git a/scripts/sbom/sbom/cli.py b/scripts/sbom/sbom/cli.py index 63c46cbe7d..b4533a96ec 100644 --- a/scripts/sbom/sbom/cli.py +++ b/scripts/sbom/sbom/cli.py @@ -193,6 +193,7 @@ def main(argv: list[str] | None = None) -> int: build_module.BuildError, licensing.CorpusError, discovery.ArchiveError, + discovery.SupplierError, wheelfile.WheelError, validate_module.ManifestError, ) as error: diff --git a/scripts/sbom/sbom/discovery.py b/scripts/sbom/sbom/discovery.py index f803127ac4..2e9b7a5404 100644 --- a/scripts/sbom/sbom/discovery.py +++ b/scripts/sbom/sbom/discovery.py @@ -149,6 +149,25 @@ def git(repo: Path, *args: str) -> str | None: ) +def _notice_from(evidence) -> str: + """The copyright a component's own licence text states, where it states one. + + Permissive licences generally require the notice to travel with the terms, + and for MIT and the BSDs it is a line of the same file -- shipped in the + wheel but, until now, named nowhere a reader could act on. + """ + # Grants only, though a pool counts as licence evidence everywhere else: a + # pool is a directory of reference texts, and the notice in one is the + # licence's own author -- OpenXR ships WTFPL, so reading its pool credited + # OpenXR to the person who wrote WTFPL. + notices = [ + found + for item in evidence + if item.kind == "grant" and (found := licensing.read_notice(item.text)) + ] + return "\n".join(licensing.fold_notices(notices)) if notices else "NOASSERTION" + + def _identity( remote: str | None, name: str, commit: str | None ) -> tuple[str, str, str, str]: @@ -232,6 +251,7 @@ def discover_source_trees(deps_dir: Path) -> dict[str, Component]: download_location=download, source_info=" ".join(details), root=root, + copyright_text=_notice_from(evidence), license_concluded=concluded, license_declared=declared_expression, evidence=tuple(evidence), @@ -272,6 +292,10 @@ def _owner(path: Path, ownership: Ownership | None) -> str | None: return ownership.owner_of(path) if ownership is not None else None +class SupplierError(Exception): + """The project does not name exactly one author to record as supplier.""" + + class ArchiveError(Exception): """An archive the build fetched cannot be read.""" @@ -506,12 +530,18 @@ def project_authors(repo_root: Path) -> set[str]: def project_supplier(repo_root: Path) -> str: """Who supplied this distribution, as its own packaging metadata states it. - SPDX takes one supplier, so several authors name nobody in particular and - none names nobody at all; either way asserting one would be this collector - speaking for a project rather than reading it. + SPDX takes one supplier. Choosing among several, or publishing a release + that names none, is a question for whoever is publishing it -- so it is + asked here rather than answered quietly. """ - authors = project_authors(repo_root) - return f"Organization: {authors.pop()}" if len(authors) == 1 else "NOASSERTION" + authors = sorted(project_authors(repo_root)) + if len(authors) != 1: + raise SupplierError( + f"{repo_root}/pyproject.toml names {len(authors)} project authors " + f"({', '.join(authors) or 'none'}); SPDX records one supplier for " + "the distribution" + ) + return f"Organization: {authors[0]}" def _holder_of(path: Path) -> tuple[str, str | None] | None: diff --git a/scripts/sbom/sbom/document.py b/scripts/sbom/sbom/document.py index 8f4afe7dbe..a73defd59e 100644 --- a/scripts/sbom/sbom/document.py +++ b/scripts/sbom/sbom/document.py @@ -259,7 +259,15 @@ def build_document( record["licenseConcluded"] = attribution.spdx_tag record["licenseInfoInFiles"] = [attribution.spdx_tag] stated_in_files.add(attribution.spdx_tag) - if attribution and attribution.copyright_text and not attribution.primary: + if ( + attribution + and attribution.copyright_text + and not attribution.primary + # A packaged licence text states its component's notice, or its own + # author's -- WTFPL names the person who wrote WTFPL. Neither is a + # notice this distribution makes about itself. + and not entry.name.startswith(f"{wheel.dist_info}/licenses/") + ): stated_copyright.add(attribution.copyright_text) comment = [] diff --git a/scripts/sbom/sbom/evidence.py b/scripts/sbom/sbom/evidence.py index 81aeefb77f..06c9749a41 100644 --- a/scripts/sbom/sbom/evidence.py +++ b/scripts/sbom/sbom/evidence.py @@ -26,6 +26,7 @@ discover_vendored, project_authors, discover_source_trees, + _notice_from, tracked_files, EMPTY_SHA256, _walk, @@ -151,6 +152,7 @@ def _archive_components( ), license_concluded=concluded, license_declared=declared, + copyright_text=_notice_from(evidence), evidence=tuple(evidence), ) return components diff --git a/scripts/sbom/sbom/inventory.py b/scripts/sbom/sbom/inventory.py index 92b0e51775..1190794cfe 100644 --- a/scripts/sbom/sbom/inventory.py +++ b/scripts/sbom/sbom/inventory.py @@ -702,19 +702,4 @@ def _spdx_tag(entry: Entry) -> str | None: """REUSE tags are upstream stating the license of that exact file.""" if Path(entry.name).suffix.lower() not in _TAGGABLE: return None - tag = licensing.read_spdx_tag(entry.head.decode("utf-8", "replace")) - if tag is None: - return None - # Tags are written by hand: lowercase operators and stray text reach the - # document as licenseConcluded, where an invalid expression fails the - # published SBOM's own conformance check. - normalized = licensing.normalized_expression(tag) - if normalized == "NOASSERTION": - return None - # A LicenseRef is only meaningful alongside its text, and a file tag carries - # none: naming one here produced a document that failed its own check for an - # undefined reference, on a wheel the build had already rewritten and would - # refuse to process again. - if licensing.license_refs(normalized): - return None - return normalized + return licensing.normalized_tag(entry.head.decode("utf-8", "replace")) diff --git a/scripts/sbom/sbom/licensing.py b/scripts/sbom/sbom/licensing.py index c39dde9cf6..e7bc6b71ff 100644 --- a/scripts/sbom/sbom/licensing.py +++ b/scripts/sbom/sbom/licensing.py @@ -523,6 +523,43 @@ def fold_notices(notices) -> list[str]: ) +_PLAIN_COPYRIGHT = re.compile( + r"^[^\w\n]*(?PCopyright\b[^\n\r]*\d{4}[^\n\r]*|" + r"Copyright\b[^\n\r]*)$", + re.MULTILINE | re.IGNORECASE, +) +_PLACEHOLDER = re.compile( + r"\[yyyy\]||\[name of copyright owner\]| str | None: + """A file's own REUSE tag, normalised, or None if it names nothing usable.""" + tag = read_spdx_tag(text) + if tag is None: + return None + expression = normalized_expression(tag) + # A LicenseRef here has no text to define it with, as for any other file. + if expression == NOASSERTION or license_refs(expression): + return None + return expression + + +def read_notice(text: str) -> str | None: + """A concrete copyright line stated in a licence text, if it carries one. + + A reference text carries the template instead -- `Copyright [yyyy] [name of + copyright owner]` -- which names nobody, so it is not a notice and recording + it would claim one where the file states none. + """ + for match in _PLAIN_COPYRIGHT.finditer(text): + notice = " ".join(match.group("notice").split()).rstrip("*/ ") + if _PLACEHOLDER.search(notice) or not re.search(r"\d{4}", notice): + continue + return notice + return None + + def read_copyright(text: str) -> str | None: """The notice a file states about itself, beside its REUSE tag.""" match = _FILE_COPYRIGHT.search(text) diff --git a/tests/python/sbom/test_discovery.py b/tests/python/sbom/test_discovery.py index a2b46b9a5f..55fbad6757 100644 --- a/tests/python/sbom/test_discovery.py +++ b/tests/python/sbom/test_discovery.py @@ -287,8 +287,8 @@ def test_the_supplier_is_read_from_the_project_not_written_here(workspace): assert discovery.project_supplier(workspace.root) == "Organization: Someone Else" -def test_a_project_naming_several_authors_names_no_single_supplier(workspace): - """SPDX takes one; picking would be this tool speaking for the project.""" +def test_a_project_naming_several_authors_stops_rather_than_choosing(workspace): + """SPDX records one supplier; which one is not this tool's call to make.""" from sbom import discovery pyproject = workspace.root / "pyproject.toml" @@ -300,4 +300,5 @@ def test_a_project_naming_several_authors_names_no_single_supplier(workspace): encoding="utf-8", ) - assert discovery.project_supplier(workspace.root) == "NOASSERTION" + with pytest.raises(discovery.SupplierError, match="names 2 project authors"): + discovery.project_supplier(workspace.root) diff --git a/tests/python/sbom/test_licensing.py b/tests/python/sbom/test_licensing.py index 407218dea7..cc5816c78d 100644 --- a/tests/python/sbom/test_licensing.py +++ b/tests/python/sbom/test_licensing.py @@ -277,3 +277,13 @@ def test_folding_does_not_merge_different_holders(): ) assert len(folded) == 2 + + +def test_a_template_is_not_a_notice(): + """A reference text names nobody; recording its placeholder would.""" + assert licensing.read_notice("Copyright [yyyy] [name of copyright owner]") is None + assert licensing.read_notice("Copyright (c) ") is None + assert ( + licensing.read_notice("MIT\n\nCopyright (c) 2016 Wenzel Jakob\n\nPermission") + == "Copyright (c) 2016 Wenzel Jakob" + ) From b0366730a00a3efe1a021fa004af9362f2f56d4e Mon Sep 17 00:00:00 2001 From: Andrei Aristarkhov Date: Thu, 1 Oct 2026 01:13:23 -0700 Subject: [PATCH 032/101] build(sbom): declare the licence a component's own file states Most components came out with licenseDeclared NOASSERTION, so a viewer reading that field showed no licence for glm, glfw, pybind11 and the rest, each of which ships a licence file plainly saying which one. That was an over-correction. The defect it fixed was real and narrow: a REUSE tag on OpenXR's COPYING.adoc -- prose about which licences the project uses, tagged CC-BY-4.0 for the prose -- was promoted to the package declaration, so a linked Apache-2.0 SDK read as documentation. The fix went further and dropped declarations wherever no tag qualified. Shipping a licence file is how authors declare a licence, so the text already read for the conclusion answers both questions, and the two coincide. That is not the field echoing: a component shipping no grant declares nothing and still says so. Tags no longer feed it at all, which is what keeps the COPYING.adoc case closed -- OpenXR declares Apache-2.0, from its LICENSE. Signed-off-by: Andrei Aristarkhov --- scripts/sbom/sbom/licensing.py | 19 ++++++++------- tests/python/sbom/test_licensing.py | 37 ++++++++++++++++++++++++----- 2 files changed, 41 insertions(+), 15 deletions(-) diff --git a/scripts/sbom/sbom/licensing.py b/scripts/sbom/sbom/licensing.py index e7bc6b71ff..fca4eed3d4 100644 --- a/scripts/sbom/sbom/licensing.py +++ b/scripts/sbom/sbom/licensing.py @@ -448,15 +448,16 @@ def expression( identified = [license_ref(component_key)] concluded = _combine(identified) if identified else "NOASSERTION" - # A tag speaks for the package only where the file carrying it is a licence - # in its own right. OpenXR's COPYING.adoc is prose about which licences the - # project uses, tagged CC-BY-4.0 for the prose and matching no licence text; - # promoting it declared a linked SDK under a documentation licence. Where no - # tag qualifies the field stays unasserted rather than echoing the - # conclusion, which would assert a declaration nobody made and hide the one - # thing the field is for -- supplier and collector disagreeing. - tags = sorted({item.spdx_tag for item in grants if item.spdx_tag and item.matches}) - declared = _combine(tags) if tags else NOASSERTION + # Declared is the same expression, because the text it was read from is the + # component's own LICENSE -- shipping that file is how authors declare a + # licence, so one artifact answers both questions. It is not an echo: a + # component that ships no grant declares nothing and gets NOASSERTION. + # + # Never from a REUSE tag on a grant file. OpenXR's COPYING.adoc is prose + # about which licences the project uses, tagged CC-BY-4.0 for the prose, and + # promoting that declared a linked Apache-2.0 SDK under a documentation + # licence. + declared = concluded if grants else NOASSERTION return concluded, declared diff --git a/tests/python/sbom/test_licensing.py b/tests/python/sbom/test_licensing.py index cc5816c78d..1464a4ea3a 100644 --- a/tests/python/sbom/test_licensing.py +++ b/tests/python/sbom/test_licensing.py @@ -185,17 +185,42 @@ def test_notices_are_packaged_but_do_not_set_the_expression(tmp_path): assert licensing.expression(evidence, "thing")[0] == "Apache-2.0" -def test_an_upstream_spdx_tag_is_reported_as_declared(tmp_path): +def test_shipping_a_license_file_is_how_authors_declare_one(tmp_path): + """Declared is not an echo of the conclusion: it is read from the same file, + because shipping that file is the declaration.""" + (tmp_path / "LICENSE").write_text(synth.spdx_text("MIT"), encoding="utf-8") + + evidence = licensing.discover_in_tree("thing", tmp_path, "thing-src") + + assert licensing.expression(evidence, "thing") == ("MIT", "MIT") + + +def test_a_component_with_no_grant_declares_nothing(tmp_path): + (tmp_path / "README.md").write_text("nothing to see", encoding="utf-8") + + evidence = licensing.discover_in_tree("thing", tmp_path, "thing-src") + + assert licensing.expression(evidence, "thing") == ("NOASSERTION", "NOASSERTION") + + +def test_a_tag_on_a_documentation_file_is_not_the_package_declaration(tmp_path): + """OpenXR's COPYING.adoc is prose about which licences the project uses, + tagged CC-BY-4.0 for the prose. Promoting it declared an Apache-2.0 SDK + under a documentation licence.""" + (tmp_path / "LICENSE").write_text(synth.spdx_text("Apache-2.0"), encoding="utf-8") # REUSE-IgnoreStart - (tmp_path / "LICENSE").write_text( - "SPDX-License-Identifier: MIT\n\n" + synth.spdx_text("MIT"), encoding="utf-8" + (tmp_path / "COPYING").write_text( + "SPDX-License-Identifier: CC-BY-4.0\n\nThis project uses several licences.\n", + encoding="utf-8", ) # REUSE-IgnoreEnd - evidence = licensing.discover_in_tree("thing", tmp_path, "thing-src") - concluded, declared = licensing.expression(evidence, "thing") + concluded, declared = licensing.expression( + licensing.discover_in_tree("thing", tmp_path, "thing-src"), "thing" + ) - assert (concluded, declared) == ("MIT", "MIT") + assert concluded == "Apache-2.0" + assert "CC-BY-4.0" not in declared def test_a_component_carrying_two_licenses_combines_them(): From f565d4896b02321c4a61a357bd44a1e17bce6e74 Mon Sep 17 00:00:00 2001 From: Andrei Aristarkhov Date: Thu, 1 Oct 2026 01:25:19 -0700 Subject: [PATCH 033/101] build(sbom): require a manifest record to name every sidecar A record was trusted for the sidecars it happened to list. Dropping the `build_evidence` entry from a manifest made `check-set` skip the evidence comparison entirely and pass the wheel, so the one artifact that binds a document to the build that produced it was optional in practice. The set of sidecars this tool writes is fixed, so a record missing one is a malformed record, not a smaller check. A wheel entry with no filename is the same error one level up: it resolved to an empty record and then to whatever `record['filename']` raised. Signed-off-by: Andrei Aristarkhov --- scripts/sbom/sbom/validate.py | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/scripts/sbom/sbom/validate.py b/scripts/sbom/sbom/validate.py index 6d17064d77..321c09e5b8 100644 --- a/scripts/sbom/sbom/validate.py +++ b/scripts/sbom/sbom/validate.py @@ -20,6 +20,9 @@ from . import wheelfile +_SIDECAR_KINDS = ("spdx", "build_evidence", "license_report_json", "license_report_md") + + class ManifestError(Exception): """A manifest is not shaped the way this tool writes them.""" @@ -47,6 +50,14 @@ def _record_for(manifest: dict, filename: str) -> dict: for key in ("sha256", "size", "sbom_in_wheel", "sidecars"): if key not in record: raise ManifestError(f"{filename}: manifest record has no {key!r}") + # Every sidecar, not just the ones a record happens to name. An + # absent entry is not an absent check: omitting `build_evidence` + # left `check-set` passing a wheel the evidence would have failed. + missing = sorted(set(_SIDECAR_KINDS) - set(record["sidecars"])) + if missing: + raise ManifestError( + f"{filename}: manifest record names no {', '.join(missing)} sidecar" + ) return record return {} @@ -454,7 +465,10 @@ def check_set(manifest_path: Path, wheel_dir: Path, evidence_dir: Path) -> list[ failures: list[str] = [] manifest = json.loads(manifest_path.read_text(encoding="utf-8")) for entry in manifest.get("wheels") or []: - record = _record_for(manifest, entry.get("filename", "")) + named = entry.get("filename") + if not named: + raise ManifestError("a manifest record names no wheel") + record = _record_for(manifest, named) wheel = wheel_dir / record["filename"] if not wheel.is_file(): failures.append(f"{record['filename']} is missing from {wheel_dir}") From ee21ed0ea4608ad46acbe540574797ad9a7a965d Mon Sep 17 00:00:00 2001 From: Andrei Aristarkhov Date: Thu, 1 Oct 2026 01:25:20 -0700 Subject: [PATCH 034/101] build(sbom): compare what a document claims with what the build recorded Verification compared licence text digests and nothing else. A digest is the one thing about a component that cannot change its terms -- which left every expression in the document free. Rewriting a CloudXR component's licenseConcluded from its LicenseRef to Apache-2.0 published a proprietary EULA as Apache-2.0 with the build evidence beside it still saying otherwise, and every gate reporting OK. The evidence records both expressions per component, so they are checkable. Two LicenseRefs are treated as agreeing: components sharing one text share one id, so the document's is not the evidence's -- and the text behind that id is the second half of this. `hasExtractedLicensingInfos` carries the verbatim terms for everything the grammar cannot name, and nothing tied those bodies back to a file either, so a body could be replaced wholesale. Where a component's terms came from several files the document joins them, leaving no single digest to compare; those stay with the packaged-path check, which reads the files themselves. Signed-off-by: Andrei Aristarkhov --- scripts/sbom/sbom/validate.py | 63 +++++++++++++++++++++++++++++++++-- 1 file changed, 60 insertions(+), 3 deletions(-) diff --git a/scripts/sbom/sbom/validate.py b/scripts/sbom/sbom/validate.py index 321c09e5b8..843a559d96 100644 --- a/scripts/sbom/sbom/validate.py +++ b/scripts/sbom/sbom/validate.py @@ -325,12 +325,12 @@ def _check_attribution(evidence_doc: dict, wheel: wheelfile.WheelInfo) -> list[s f"{key} is redistributed but the license text recorded for it is " "not packaged in this wheel" ) - failures.extend(_document_agrees_with_evidence(wheel, recorded)) + failures.extend(_document_agrees_with_evidence(wheel, recorded, evidence_doc)) return failures def _document_agrees_with_evidence( - wheel: wheelfile.WheelInfo, recorded: set[str] + wheel: wheelfile.WheelInfo, recorded: set[str], evidence_doc: dict ) -> list[str]: """The texts the document names must be the ones the build actually read. @@ -356,11 +356,68 @@ def _document_agrees_with_evidence( f"{package['name']}: the document names a licence text " f"(sha256:{match.group('digest')}) the build never read" ) + + # A digest is the one thing about a component that cannot change its terms. + # Comparing only that left every expression in the document rewritable -- + # a proprietary EULA could be published as Apache-2.0, with the evidence + # beside it still saying otherwise and every gate reporting OK. + components = evidence_doc.get("components", {}) + by_name = { + component.get("name", key): (key, component) + for key, component in components.items() + } + for package in spdx.get("packages", []): + found = by_name.get(package["name"]) + if found is None: + continue + key, component = found + for field, stated in ( + ("licenseConcluded", "license_concluded"), + ("licenseDeclared", "license_declared"), + ): + expected = component.get(stated) + actual = package.get(field) + if not expected or actual == expected: + continue + # Both unnameable is agreement: components sharing one text share one + # LicenseRef, so the document's id is not the evidence's. The text + # behind it is checked below, which is what the id stands for. + if "LicenseRef-" in str(actual) and "LicenseRef-" in expected: + continue + failures.append( + f"{key}: the document says {field} {actual!r}, " + f"the build recorded {expected!r}" + ) + + # And the text that id stands for has to be one the build read. Where a + # component's terms came from several files the document joins them, so + # there is no single digest to compare and the join is left to the + # packaged-path check. + single = { + item["sha256"] + for component in components.values() + for item in [component.get("evidence", [])] + if len(item) == 1 + for item in item + } + for extracted in spdx.get("hasExtractedLicensingInfos", []): + digest = hashlib.sha256( + extracted.get("extractedText", "").encode("utf-8") + ).hexdigest() + if single and digest not in recorded and digest not in single: + failures.append( + f"{extracted['licenseId']}: the text it carries is not one the " + "build read" + ) return failures def _refs_in(value) -> set[str]: - """LicenseRef identifiers in a field that may hold an expression or a list.""" + """LicenseRef identifiers in a field that may hold an expression or a list. + + An expression that cannot be read is reported as such rather than treated as + naming nothing. + """ if value is None: return set() items = value if isinstance(value, list) else [value] From 8740382b71cf987a64ecbee545d9e6b32569dafe Mon Sep 17 00:00:00 2001 From: Andrei Aristarkhov Date: Thu, 1 Oct 2026 01:25:21 -0700 Subject: [PATCH 035/101] build(sbom): count every text the build read as read The document cites one attributionText per evidence item, but verification built its set of known digests from the licence-granting items only. A component whose evidence included a NOTICE failed for naming a text the build had plainly read. Signed-off-by: Andrei Aristarkhov --- scripts/sbom/sbom/validate.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/scripts/sbom/sbom/validate.py b/scripts/sbom/sbom/validate.py index 843a559d96..a05ed1fe5e 100644 --- a/scripts/sbom/sbom/validate.py +++ b/scripts/sbom/sbom/validate.py @@ -297,11 +297,10 @@ def _check_attribution(evidence_doc: dict, wheel: wheelfile.WheelInfo) -> list[s for key, component in evidence_doc.get("components", {}).items(): if not component.get("in_this_wheel"): continue - recorded.update( - item["sha256"] - for item in component.get("evidence", []) - if item["kind"] in ("grant", "pool") - ) + # Every kind: the document cites one attributionText per evidence item, + # so comparing only the licence-granting ones called a NOTICE the build + # plainly read a text it never did. + recorded.update(item["sha256"] for item in component.get("evidence", [])) # The same test the build gate applies: a REUSE pool is a real text even # though it names no single expression, so publishing on one and then # failing verification for want of a grant would contradict the gate. From 93b78d7016594b00513f400fb8fb46df10deffc4 Mon Sep 17 00:00:00 2001 From: Andrei Aristarkhov Date: Thu, 1 Oct 2026 01:25:23 -0700 Subject: [PATCH 036/101] build(sbom): check the files a package claims to hold are here `hasFiles` names the document's own file entries. Nothing checked the endpoints, so a package could claim files the document does not describe and a reader resolving them would find nothing. Signed-off-by: Andrei Aristarkhov --- scripts/sbom/sbom/validate.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/scripts/sbom/sbom/validate.py b/scripts/sbom/sbom/validate.py index a05ed1fe5e..41ce41546a 100644 --- a/scripts/sbom/sbom/validate.py +++ b/scripts/sbom/sbom/validate.py @@ -443,6 +443,13 @@ def _check_licenses(wheel: wheelfile.WheelInfo, spdx: dict) -> list[str]: for package in spdx["packages"]: for field in ("licenseConcluded", "licenseDeclared", "licenseInfoFromFiles"): declared_refs.update(_refs_in(package.get(field))) + # Every file id a package claims to hold has to be a file this document has. + file_ids = {item["SPDXID"] for item in spdx.get("files", [])} + for package in spdx["packages"]: + for held in package.get("hasFiles", []): + if held not in file_ids: + failures.append(f"{package['name']} claims {held}, which is not here") + extracted = { item["licenseId"] for item in spdx.get("hasExtractedLicensingInfos", []) } From b11de4d84bdc7294deabb0c46f325238886318fe Mon Sep 17 00:00:00 2001 From: Andrei Aristarkhov Date: Thu, 1 Oct 2026 01:25:39 -0700 Subject: [PATCH 037/101] build(sbom): fail on an expression that cannot be read A parse failure returned no identifiers, which reads as `this expression uses no LicenseRef`. The check that every reference is defined then passed vacuously: a document carrying `LicenseRef-never-defined AND (((` was reported as defining everything it uses. An unreadable expression means what it names is unknown, which is a failure and not an absence. Signed-off-by: Andrei Aristarkhov --- scripts/sbom/sbom/cli.py | 1 + scripts/sbom/sbom/licensing.py | 11 +++++++++-- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/scripts/sbom/sbom/cli.py b/scripts/sbom/sbom/cli.py index b4533a96ec..0b4e11c6e7 100644 --- a/scripts/sbom/sbom/cli.py +++ b/scripts/sbom/sbom/cli.py @@ -192,6 +192,7 @@ def main(argv: list[str] | None = None) -> int: evidence_module.EvidenceError, build_module.BuildError, licensing.CorpusError, + licensing.ExpressionReadError, discovery.ArchiveError, discovery.SupplierError, wheelfile.WheelError, diff --git a/scripts/sbom/sbom/licensing.py b/scripts/sbom/sbom/licensing.py index fca4eed3d4..91d3b266b9 100644 --- a/scripts/sbom/sbom/licensing.py +++ b/scripts/sbom/sbom/licensing.py @@ -606,14 +606,21 @@ def canonical_evidence( return found +class ExpressionReadError(Exception): + """An expression in a document cannot be read, so what it names is unknown.""" + + def license_refs(*expressions: str) -> list[str]: """The LicenseRef- identifiers an expression uses, per the SPDX grammar.""" found: set[str] = set() for expression in expressions: try: parsed = _spdx_licensing().parse(expression, validate=False) - except (ExpressionError, ValueError, TypeError): - continue + except (ExpressionError, ValueError, TypeError) as error: + # Walking past it hid every reference inside: a document could carry + # `LicenseRef-never-defined AND (((` and be reported as defining + # everything it uses. + raise ExpressionReadError(f"{expression!r}: {error}") from error found.update( key for key in _spdx_licensing().license_keys(parsed) From dccd929b28bda6de5e694ecf66e2eeef105bfae8 Mon Sep 17 00:00:00 2001 From: Andrei Aristarkhov Date: Thu, 1 Oct 2026 01:25:58 -0700 Subject: [PATCH 038/101] build(sbom): reject a file tag the grammar cannot name `normalized_expression` parses without validating, so a REUSE tag naming an identifier newer than the shipped grammar landed on a file record and then failed the document's own conformance check. A conclusion is already tested this way; a per-file tag is the same question. Signed-off-by: Andrei Aristarkhov --- scripts/sbom/sbom/licensing.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/scripts/sbom/sbom/licensing.py b/scripts/sbom/sbom/licensing.py index 91d3b266b9..7d1940de3a 100644 --- a/scripts/sbom/sbom/licensing.py +++ b/scripts/sbom/sbom/licensing.py @@ -543,6 +543,11 @@ def normalized_tag(text: str) -> str | None: # A LicenseRef here has no text to define it with, as for any other file. if expression == NOASSERTION or license_refs(expression): return None + # And the grammar has to know every id in it. `normalized_expression` parses + # without validating, so a tag naming nothing real reached the document and + # failed its own conformance check -- the same reason a conclusion is tested. + if not all(map(expressible, _spdx_licensing().license_keys(expression))): + return None return expression From c2cb872b4bb1253418c6f86226a9941a7412ca7d Mon Sep 17 00:00:00 2001 From: Andrei Aristarkhov Date: Thu, 1 Oct 2026 01:26:04 -0700 Subject: [PATCH 039/101] build(sbom): do not credit a component with the licence steward's notice A COPYING that *is* the reference text carries the steward's own copyright line -- the FSF's in every GPL, Sam Hocevar's in WTFPL -- so a component shipping one verbatim was attributed to whoever wrote the licence. The corpus already holds those texts, so the question is answered by looking at the matched reference rather than by keeping a list of stewards, which would go stale the next time the list grows. Signed-off-by: Andrei Aristarkhov --- scripts/sbom/sbom/discovery.py | 6 +++++- scripts/sbom/sbom/licensing.py | 18 ++++++++++++++++++ 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/scripts/sbom/sbom/discovery.py b/scripts/sbom/sbom/discovery.py index 2e9b7a5404..84a08fe320 100644 --- a/scripts/sbom/sbom/discovery.py +++ b/scripts/sbom/sbom/discovery.py @@ -163,7 +163,11 @@ def _notice_from(evidence) -> str: notices = [ found for item in evidence - if item.kind == "grant" and (found := licensing.read_notice(item.text)) + if item.kind == "grant" + and (found := licensing.read_notice(item.text)) + and not licensing.notice_is_the_licence_authors( + found, [license_id for license_id, _, _ in item.matches] + ) ] return "\n".join(licensing.fold_notices(notices)) if notices else "NOASSERTION" diff --git a/scripts/sbom/sbom/licensing.py b/scripts/sbom/sbom/licensing.py index 7d1940de3a..f822a74c1d 100644 --- a/scripts/sbom/sbom/licensing.py +++ b/scripts/sbom/sbom/licensing.py @@ -551,6 +551,24 @@ def normalized_tag(text: str) -> str | None: return expression +def notice_is_the_licence_authors(notice: str, identified: list[str]) -> bool: + """Whether this notice belongs to the licence rather than to the component. + + A COPYING that *is* the reference text carries the steward's own line -- the + FSF's in every GPL, Sam Hocevar's in WTFPL -- so a component shipping one + verbatim would be credited to whoever wrote the licence. The corpus already + holds those texts, so the question is answered by looking rather than by + keeping a list of stewards. + """ + entries = corpus()[1] + collapsed = " ".join(notice.split()) + return any( + (reference := entries.get(license_id)) + and collapsed in " ".join(reference.text.split()) + for license_id in identified + ) + + def read_notice(text: str) -> str | None: """A concrete copyright line stated in a licence text, if it carries one. From 82fe9ad9e4d47cda0a743270cd332402e7ea1925 Mon Sep 17 00:00:00 2001 From: Andrei Aristarkhov Date: Thu, 1 Oct 2026 01:26:05 -0700 Subject: [PATCH 040/101] build(sbom): give a system library the notices its copyright file states The distro copyright file was read, packaged and put on the file record, and then left off the package and out of the notices every other component appears in. It is where the holder is stated, so a reader looking up who owns a linked system library found NOASSERTION. Signed-off-by: Andrei Aristarkhov --- scripts/sbom/sbom/discovery.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/scripts/sbom/sbom/discovery.py b/scripts/sbom/sbom/discovery.py index 84a08fe320..de5bfff21b 100644 --- a/scripts/sbom/sbom/discovery.py +++ b/scripts/sbom/sbom/discovery.py @@ -865,6 +865,10 @@ def system_component(soname: str, origin: dict) -> Component: name=soname, kind="system-library", supplier=origin.get("supplier") or "NOASSERTION", + # The distro copyright file is where the holder is stated, and it was + # read, packaged and put on the file record -- then left off the package + # and out of the notices every other component appears in. + copyright_text=_notice_from(evidence), homepage="NOASSERTION", # A distro package the machine can name resolves; pkg:generic does not. purl=( From 59adfd2995c0060d524e0b8aa4e9d58fba0a1c89 Mon Sep 17 00:00:00 2001 From: Andrei Aristarkhov Date: Thu, 1 Oct 2026 01:26:07 -0700 Subject: [PATCH 041/101] build(sbom): read licence terms only from the texts packaged as evidence Members the collector adds were all parsed for a tag and a notice, including its own output. Reading the notices file back found the first `- Copyright:` bullet of the summary it had just generated and recorded it as that file's copyright. Only the third-party texts are evidence of anything. Signed-off-by: Andrei Aristarkhov --- scripts/sbom/sbom/build.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/scripts/sbom/sbom/build.py b/scripts/sbom/sbom/build.py index d3ddf6f0ff..69adea4567 100644 --- a/scripts/sbom/sbom/build.py +++ b/scripts/sbom/sbom/build.py @@ -152,6 +152,10 @@ def build( ] projected_wheel = replace(wheel, entries=tuple(projected)) for name in [*additions, *replacements]: + # Only the third-party texts: the notices file and the document below + # are this collector's own output, and reading them back found the first + # `- Copyright:` bullet of the summary it had just generated. + packaged_text = name in additions and "/licenses/third-party/" in name body = additions.get(name, b"").decode("utf-8", "replace") # A licence text can state its own terms and carry a notice, and these # were the only members never read for either -- the collector packaged @@ -163,8 +167,8 @@ def build( detail="license evidence packaged by the SBOM collector" if name in additions else "distribution metadata, updated to declare the packaged license files", - spdx_tag=licensing.normalized_tag(body) if name in additions else None, - copyright_text=licensing.read_notice(body) if name in additions else None, + spdx_tag=licensing.normalized_tag(body) if packaged_text else None, + copyright_text=licensing.read_notice(body) if packaged_text else None, ) # The document is added below, after it has been rendered; record it now so # the evidence accounts for every member of the finished wheel. From 8fa9cb233cbbf8ef42211e5fe6fd06d33fc35a4f Mon Sep 17 00:00:00 2001 From: Andrei Aristarkhov Date: Thu, 1 Oct 2026 01:26:08 -0700 Subject: [PATCH 042/101] build(sbom): keep the path separators in a packaged licence's name The packaged path is named after a text's whole path within its component precisely so that `LICENSE` and `LICENSES/MIT.txt` do not collide. Replacing the separators flattened them back into one name: the second write lost, and the document went on naming the path it was dropped from. Signed-off-by: Andrei Aristarkhov --- scripts/sbom/sbom/report.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/scripts/sbom/sbom/report.py b/scripts/sbom/sbom/report.py index 58e22beadb..eb3a6f1264 100644 --- a/scripts/sbom/sbom/report.py +++ b/scripts/sbom/sbom/report.py @@ -45,7 +45,11 @@ def packaged_license_path(dist_info: str, component_key: str, evidence) -> str: """ folder = _SAFE_NAME.sub("_", component_key) relative = evidence.path.rsplit("!", 1)[-1].split("/", 1)[-1] - return f"{dist_info}/licenses/third-party/{folder}/{_SAFE_NAME.sub('_', relative)}" + # Keep the separators: flattening them re-created the collision this whole + # scheme exists to avoid, and the loser was dropped while the document went + # on naming the path it was dropped from. + safe = "/".join(_SAFE_NAME.sub("_", part) for part in relative.split("/")) + return f"{dist_info}/licenses/third-party/{folder}/{safe}" def _roles(inventory: Inventory, key: str) -> str: From da45113c4f927fdab53e4966960997eca7691b55 Mon Sep 17 00:00:00 2001 From: Andrei Aristarkhov Date: Thu, 1 Oct 2026 01:26:10 -0700 Subject: [PATCH 043/101] build(sbom): state one node per required package and range A dependency recurs once per extra that wants it, so a wheel's document listed the same package over and over with the same unasserted version and the same range, differing only in a marker. A viewer showed a column of identical rows. The repeats are one dependency. The extras that ask for it go in the comment, and the relationship stays OPTIONAL_DEPENDENCY_OF only when every line asking for it is gated -- one unconditional line makes the dependency unconditional. Signed-off-by: Andrei Aristarkhov --- scripts/sbom/sbom/document.py | 36 ++++++++++++++++++++--------------- 1 file changed, 21 insertions(+), 15 deletions(-) diff --git a/scripts/sbom/sbom/document.py b/scripts/sbom/sbom/document.py index a73defd59e..b67143b0bf 100644 --- a/scripts/sbom/sbom/document.py +++ b/scripts/sbom/sbom/document.py @@ -413,16 +413,20 @@ def build_document( } ) + # One node per package and version range, not per requirement line. A + # dependency recurs once per extra that wants it, and the repeats differ + # only in a marker the relationship already carries. + wanted: dict[tuple[str, str], list[Requirement]] = {} for requirement in inventory.requires_dist: parsed = Requirement(requirement) - name = parsed.name - # Not the requirement string: slugifying `websockets>=14.0` yields - # `websockets-14.0`, which reads as a pinned version beside a - # versionInfo this document deliberately leaves unasserted. The same - # name recurs under different extras, so uniqueness comes from a digest - # of the whole requirement; its text is in the comment below. - digest = hashlib.sha256(requirement.encode("utf-8")).hexdigest()[:8] - package_id = _spdx_id("Package-pypi", f"{name}-{digest}") + wanted.setdefault((parsed.name, str(parsed.specifier)), []).append(parsed) + + for (name, _), group in sorted(wanted.items()): + package_id = _spdx_id( + "Package-pypi", + f"{name}-{hashlib.sha256(str(sorted(map(str, group))).encode()).hexdigest()[:8]}", + ) + extras = sorted({extra for item in group for extra in item.extras}) packages.append( { "SPDXID": package_id, @@ -434,17 +438,19 @@ def build_document( "licenseConcluded": NOASSERTION, "licenseDeclared": NOASSERTION, "copyrightText": NOASSERTION, - "externalRefs": _dependency_refs(name, parsed), + "externalRefs": _dependency_refs(name, group[0]), "comment": ( - f"Consumer requirement declared in wheel metadata: {requirement}. " - "Resolved at install time; this document states no license for it." + "Consumer requirement declared in wheel metadata: " + + "; ".join(sorted(str(item) for item in group)) + + ". Resolved at install time; this document states no " + "license for it." + + (f" Requested with: {', '.join(extras)}." if extras else "") ), } ) - # An `extra ==` marker means the requirement only applies when that extra - # is requested, which SPDX has a relationship for. Its direction is the - # reverse of DEPENDS_ON. - if _is_extra_gated(parsed): + # Optional only if every line that asks for it is gated on an extra. One + # unconditional line makes the dependency unconditional. + if all(_is_extra_gated(item) for item in group): relationships.append( { "spdxElementId": package_id, From 71476b1bfa1affa59ff6fb72178b2cf421c2fadf Mon Sep 17 00:00:00 2001 From: Andrei Aristarkhov Date: Thu, 1 Oct 2026 01:26:11 -0700 Subject: [PATCH 044/101] docs(agents): do not state counts that drift in a comment Comments here stated how many files a route covered, how many build-ids two SDKs shared, how many places a rule was written in. Each was true the day it was written and nothing fails when it goes stale, so the next reader is told a number the code no longer supports. The property is the durable part: these SDKs share build-ids. The existing `keep measured numbers` line pulled the other way, which is why the habit kept coming back. Signed-off-by: Andrei Aristarkhov --- AGENTS.md | 7 +++++++ scripts/sbom/sbom/document.py | 2 +- scripts/sbom/sbom/inventory.py | 2 +- 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 2e28c6cfeb..06752ccd75 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -171,6 +171,13 @@ measurement, a trap someone already paid for. They are not a place to narrate. inertial frame, so STL axes need `mesh_pos`/`mesh_quat`" beats three paragraphs re-deriving why. Keep measured numbers and file/line references; cut the prose around them. +- **No counts that drift.** A comment must not state how many files, rows, + components or call sites something currently has. The number is true the day + it is written and wrong after the next build or refactor, and nothing fails + when it goes stale. State the property instead — "these SDKs share build-ids", + not "they share five". A measured number earns its place only when it is a + fixed fact about the world (a protocol limit, a benchmark with its date). + - **Do not narrate history.** "An earlier revision did X and it was wrong" is what `git log` is for. If the wrong approach is tempting enough to warn about, write the warning as a rule — "do not derive this from the mesh" — diff --git a/scripts/sbom/sbom/document.py b/scripts/sbom/sbom/document.py index b67143b0bf..b4dbfd7f50 100644 --- a/scripts/sbom/sbom/document.py +++ b/scripts/sbom/sbom/document.py @@ -93,7 +93,7 @@ def _extracted_licenses( component.license_concluded, component.license_declared ) # Same predicate as the build gate, the verifier and the report: a REUSE - # pool is a real text. Written four ways, it was a rule in four places. + # pool is a real text. Change one of those and change all of them. grants = [item for item in component.evidence if item.kind in ("grant", "pool")] if not refs or not grants: continue diff --git a/scripts/sbom/sbom/inventory.py b/scripts/sbom/sbom/inventory.py index 1190794cfe..dc857b1303 100644 --- a/scripts/sbom/sbom/inventory.py +++ b/scripts/sbom/sbom/inventory.py @@ -407,7 +407,7 @@ def _resolve_by_build_id( ) -> Attribution | None: """The pristine bytes this member was patched from, named by build-id.""" # The same library can sit in two archives under two sets of terms, and - # these two SDKs share five build-ids. Every neighbouring index refuses + # SDKs in this build do share build-ids. Every neighbouring index refuses # to name one of several candidates; this one must too, or a patched # member takes whichever archive happened to be walked first. members = self.archives.all_by_build_id.get(build_id, []) From 8753e75349b3d9e03162628fc3c36057c904b142 Mon Sep 17 00:00:00 2001 From: Andrei Aristarkhov Date: Thu, 1 Oct 2026 01:34:50 -0700 Subject: [PATCH 045/101] build(sbom): keep the staged tree out of the index whatever --build-dir says The exclusion that holds the staged package tree out of the build index resolved its skip paths to absolute, then compared them against the paths the walk yields -- which are relative when `--build-dir` is. Nothing matched, so the staged tree was indexed. The workflow passes `--build-dir build`, so this was the live configuration. Indexing the tree a wheel is built from lets a member explain itself from its own staged twin, which is the one route around the gate every other member has to pass; it also made a build-tree path ambiguous, which is how it surfaced -- `isaaccapture/py.typed` matched three staged copies and resolved to none. The comparison now happens inside the tree being walked. A string prefix was wrong for a second reason while it lasted: `_deps/foo-srcx` starts with `_deps/foo-src`. Signed-off-by: Andrei Aristarkhov --- scripts/sbom/sbom/discovery.py | 17 +++++++++++++++-- tests/python/sbom/test_discovery.py | 24 ++++++++++++++++++++++++ 2 files changed, 39 insertions(+), 2 deletions(-) diff --git a/scripts/sbom/sbom/discovery.py b/scripts/sbom/sbom/discovery.py index de5bfff21b..07acebea3c 100644 --- a/scripts/sbom/sbom/discovery.py +++ b/scripts/sbom/sbom/discovery.py @@ -1060,14 +1060,27 @@ def digest_of(self, path: Path) -> str | None: def add_tree( self, root: Path, display_root: str, skip: tuple[Path, ...] = () ) -> None: - skipped = tuple(item.resolve() for item in skip) + # Compare inside the tree being walked. `--build-dir` is free-form, and + # a relative root yields relative paths that match no absolute skip + # entry, so passing `--build-dir build` -- what the workflow passes -- + # indexed the staged tree and let a wheel member explain itself. A + # string prefix was wrong for a second reason: `_deps/foo-srcx` starts + # with `_deps/foo-src`. + inside = [] + for item in skip: + try: + inside.append(item.resolve().relative_to(root.resolve())) + except ValueError: + continue self._add( root, display_root, ( path for path in _walk(root) - if not any(str(path).startswith(str(item)) for item in skipped) + if not any( + path.relative_to(root).is_relative_to(item) for item in inside + ) ), ) diff --git a/tests/python/sbom/test_discovery.py b/tests/python/sbom/test_discovery.py index 55fbad6757..3eae431ee8 100644 --- a/tests/python/sbom/test_discovery.py +++ b/tests/python/sbom/test_discovery.py @@ -5,6 +5,8 @@ from __future__ import annotations +from pathlib import Path + import pytest import synth from sbom import discovery @@ -302,3 +304,25 @@ def test_a_project_naming_several_authors_stops_rather_than_choosing(workspace): with pytest.raises(discovery.SupplierError, match="names 2 project authors"): discovery.project_supplier(workspace.root) + + +def test_a_relative_build_dir_still_keeps_the_staged_tree_out( + workspace, license_data, monkeypatch +): + """`--build-dir` is free-form, and the workflow passes a relative one. + + The staged package tree is what the wheel was built from, so indexing it + lets a member explain itself -- the one route around the gate every other + member has to pass. The exclusion compared an absolute skip path against + relative walked ones and silently matched nothing. + """ + from sbom import evidence, licensing + + licensing.load_corpus(license_data) + monkeypatch.chdir(workspace.root) + + discovered = evidence.discover(workspace.root, Path("build")) + indexed = set(discovered.build_files.by_hash.values()) + + assert indexed, "the build tree is indexed" + assert not any("/python_package/" in item for item in indexed) From 544240544df857e0e3923d79d86f4b9c43009e9d Mon Sep 17 00:00:00 2001 From: Andrei Aristarkhov Date: Thu, 1 Oct 2026 01:34:56 -0700 Subject: [PATCH 046/101] build(sbom): separate joined requirements by something they do not contain A requirement string carries its own semicolon before the marker, so joining a collapsed group with "; " produced a comment that reads as one longer requirement list than it is. Signed-off-by: Andrei Aristarkhov --- scripts/sbom/sbom/document.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/scripts/sbom/sbom/document.py b/scripts/sbom/sbom/document.py index b4dbfd7f50..8172492aaf 100644 --- a/scripts/sbom/sbom/document.py +++ b/scripts/sbom/sbom/document.py @@ -441,7 +441,9 @@ def build_document( "externalRefs": _dependency_refs(name, group[0]), "comment": ( "Consumer requirement declared in wheel metadata: " - + "; ".join(sorted(str(item) for item in group)) + # " | ", not "; ": a requirement string contains its own + # semicolon before the marker. + + " | ".join(sorted(str(item) for item in group)) + ". Resolved at install time; this document states no " "license for it." + (f" Requested with: {', '.join(extras)}." if extras else "") From 94a14ddb5ac683d24e84f35a7295231d53263fb6 Mon Sep 17 00:00:00 2001 From: Andrei Aristarkhov Date: Thu, 1 Oct 2026 01:47:13 -0700 Subject: [PATCH 047/101] refactor(sbom): give each shared rule one place to live Four modules each wrote out "a grant or a REUSE pool discharges the packaging obligation", three of them with a comment saying the other three had to agree. The set already drifted once. It is `licensing.OBLIGATION_KINDS` now, and the comments are gone with the duplication. The same move for four more rules that were stated twice or more: where licence texts land inside a wheel (the writer and the gate built the same literal independently), which component a `FileIndex` display path belongs to (its own rendering convention, parsed back by string surgery in two other modules), the host library search roots, and the dpkg query. `record_hash` now goes through `record_hash_from_sha256` rather than spelling PEP 376's encoding out a second time, and `build.sha256_file` was a byte-identical copy of `discovery`'s. `_classify` and `_evidence` are public: five call sites in two modules reached for them under a lint waiver, which is the module saying its surface was wrong. The vendored scan's EULA branch went with them -- `_GRANT_NAMES` has matched EULA all along -- and the candidate set it feeds is now a `BuildGraph` method instead of a reach into `_nodes`. Vendored files are read once for holder, tag and notice rather than opened twice for the same 4096 bytes, and `/etc/os-release` goes through `platform.freedesktop_os_release`, which handles the quoting the hand parser did not. Signed-off-by: Andrei Aristarkhov --- scripts/sbom/sbom/build.py | 38 +++------ scripts/sbom/sbom/discovery.py | 116 +++++++++++++++++----------- scripts/sbom/sbom/document.py | 6 +- scripts/sbom/sbom/evidence.py | 21 ++--- scripts/sbom/sbom/inventory.py | 7 +- scripts/sbom/sbom/licensing.py | 30 +++++-- scripts/sbom/sbom/report.py | 36 ++++++++- scripts/sbom/sbom/validate.py | 13 ++-- scripts/sbom/sbom/wheelfile.py | 16 +--- tests/python/sbom/test_licensing.py | 4 +- 10 files changed, 160 insertions(+), 127 deletions(-) diff --git a/scripts/sbom/sbom/build.py b/scripts/sbom/sbom/build.py index 69adea4567..651759171c 100644 --- a/scripts/sbom/sbom/build.py +++ b/scripts/sbom/sbom/build.py @@ -20,7 +20,7 @@ from . import inventory as inventory_module from . import report as report_module from . import wheelfile -from .discovery import project_supplier +from .discovery import project_supplier, sha256_file from .inventory import Attribution from .wheelfile import Entry @@ -47,24 +47,6 @@ def _entry(name: str, data: bytes) -> Entry: ) -def _declared_path(dist_info: str, member: str) -> str: - """What METADATA calls a packaged license file. - - PEP 639 states License-File relative to `.dist-info/licenses/`, which is - what setuptools already does for the project's own texts; declaring the - member path instead yields an entry that resolves to nothing. - """ - return member.removeprefix(f"{dist_info}/licenses/") - - -def sha256_file(path: Path) -> str: - digest = hashlib.sha256() - with path.open("rb") as handle: - for chunk in iter(lambda: handle.read(1 << 20), b""): - digest.update(chunk) - return digest.hexdigest() - - def default_license_data(build_dir: Path) -> Path: """Where deps/third_party materializes the SPDX license list for this build.""" return build_dir / "_deps" / "license-list-data-src" / "json" @@ -103,12 +85,10 @@ def build( + "\nThey reached the wheel by a route the collector cannot see." ) - # A REUSE LICENSES/ pool is a real text even though it names no single - # expression for the component, so it satisfies the packaging obligation. missing = [ key for key, component in inventory.components_present.items() - if not any(item.kind in ("grant", "pool") for item in component.evidence) + if not licensing.obligation_texts(component.evidence) ] if missing: raise BuildError( @@ -128,13 +108,17 @@ def build( if target in additions: continue additions[target] = item.text.encode("utf-8") - packaged_paths.append(_declared_path(wheel.dist_info, target)) + packaged_paths.append( + report_module.declared_license_path(wheel.dist_info, target) + ) - notices_path = f"{wheel.dist_info}/licenses/THIRD-PARTY-NOTICES.md" + notices_path = report_module.notices_path(wheel.dist_info) additions[notices_path] = report_module.notices_markdown( inventory, wheel.dist_info, wheel_path.name ).encode("utf-8") - packaged_paths.append(_declared_path(wheel.dist_info, notices_path)) + packaged_paths.append( + report_module.declared_license_path(wheel.dist_info, notices_path) + ) patched_metadata = wheelfile.add_license_files(metadata_raw, sorted(packaged_paths)) metadata_declared = patched_metadata != metadata_raw @@ -155,7 +139,9 @@ def build( # Only the third-party texts: the notices file and the document below # are this collector's own output, and reading them back found the first # `- Copyright:` bullet of the summary it had just generated. - packaged_text = name in additions and "/licenses/third-party/" in name + packaged_text = name in additions and report_module.is_third_party_license( + wheel.dist_info, name + ) body = additions.get(name, b"").decode("utf-8", "replace") # A licence text can state its own terms and carry a notice, and these # were the only members never read for either -- the collector packaged diff --git a/scripts/sbom/sbom/discovery.py b/scripts/sbom/sbom/discovery.py index 07acebea3c..b534d16756 100644 --- a/scripts/sbom/sbom/discovery.py +++ b/scripts/sbom/sbom/discovery.py @@ -21,6 +21,7 @@ import glob import hashlib import json +import platform import re import shutil import subprocess @@ -64,6 +65,16 @@ def sha256_bytes(data: bytes) -> str: return hashlib.sha256(data).hexdigest() +def component_of(display: str) -> str: + """The component a `FileIndex` display path belongs to. + + `add_tree` renders `/` and a fetched checkout's tree is + `-src`, so this is the inverse of how the index was populated. Inverting + it at the call site made the convention an undeclared contract. + """ + return display.split("/", 1)[0].removesuffix("-src") + + def sha256_file(path: Path) -> str: digest = hashlib.sha256() with path.open("rb") as handle: @@ -377,6 +388,29 @@ def resolve(raw: str, root: Path) -> Path: def by_name(self, name: str) -> list[Artifact]: return self._by_name.get(name, []) + def repo_inputs(self, repo_root: Path, build_dir: Path) -> set[Path]: + """Files under `repo_root` this build compiled or included. + + `_deps` is the component-source domain and is attributed from its own + checkouts; an include directory under the build tree holds generated + headers, which state no upstream holder. + """ + found: set[Path] = set() + for node in self._nodes.values(): + found.update( + source + for source in node.sources + if str(source).startswith(str(repo_root)) + and "/_deps/" not in source.as_posix() + ) + for include in node.includes: + text = include.as_posix() + if not text.startswith(str(repo_root)) or "/_deps/" in text: + continue + if include.is_dir() and not text.startswith(str(build_dir)): + found.update(item for item in include.rglob("*") if item.is_file()) + return found + def _closure(self, target_id: str) -> list[_Node]: """The target and everything it depends on, transitively.""" seen: set[str] = set() @@ -548,8 +582,17 @@ def project_supplier(repo_root: Path) -> str: return f"Organization: {authors[0]}" -def _holder_of(path: Path) -> tuple[str, str | None] | None: - """The copyright holder and licence a file states about itself.""" +@dataclass(frozen=True) +class _Stated: + """What a source file says about itself in its header.""" + + holder: str + expression: str | None + notice: str + + +def _stated_by(path: Path) -> _Stated | None: + """Holder, licence and notice, from one read of the file's head.""" try: head = path.open("rb").read(4096).decode("utf-8", "replace") except OSError: @@ -559,21 +602,12 @@ def _holder_of(path: Path) -> tuple[str, str | None] | None: return None holder = _YEARS.sub(" ", match.group("holder") or match.group("plain") or "") holder = " ".join(holder.split()).removesuffix(" All rights reserved").strip() - return (holder, licensing.read_spdx_tag(head)) if holder else None - - -def _notice_of(path: Path) -> str | None: - """The copyright line a file states, as written.""" - try: - head = path.open("rb").read(4096).decode("utf-8", "replace") - except OSError: - return None - match = _COPYRIGHT_TAG.search(head) - if not match: + if not holder: return None notice = " ".join(match.group(0).split()).lstrip("/ *#") # The tag introduces the notice; it is not part of it. - return re.sub(r"^SPDX-FileCopyrightText:\s*", "", notice) + notice = re.sub(r"^SPDX-FileCopyrightText:\s*", "", notice) + return _Stated(holder, licensing.read_spdx_tag(head), notice) def discover_vendored( @@ -586,18 +620,17 @@ def discover_vendored( notices: dict[str, set[str]] = {} for path in sorted(candidates): - stated = _holder_of(path) + stated = _stated_by(path) if stated is None: continue - holder, expression = stated - if any(author.lower() in holder.lower() for author in authors): + if any(author.lower() in stated.holder.lower() for author in authors): continue - key = f"vendored:{licensing.spdx_safe(holder).lower()}" + key = f"vendored:{licensing.spdx_safe(stated.holder).lower()}" grouped.setdefault(key, []).append(path) - names[key] = holder - notices.setdefault(key, set()).add(_notice_of(path) or holder) - if expression: - licences.setdefault(key, set()).add(expression) + names[key] = stated.holder + notices.setdefault(key, set()).add(stated.notice) + if stated.expression: + licences.setdefault(key, set()).add(stated.expression) components: dict[str, Component] = {} for key, paths in grouped.items(): @@ -668,8 +701,7 @@ def _host_library_paths() -> list[Path]: paths.add(Path(target)) except (OSError, subprocess.SubprocessError): pass - roots = [Path(item) for item in _SEARCH_ROOTS if Path(item).is_dir()] - roots += [Path(item) for item in sorted(glob.glob("/usr/local/cuda*/lib64"))] + roots = _library_roots() for root in roots: paths.update(item for item in root.glob("*.so*") if item.is_file()) return sorted(paths) @@ -724,8 +756,7 @@ def resolve_system_library(soname: str) -> dict: "package": None, "copyright": None, } - roots = [Path(item) for item in _SEARCH_ROOTS if Path(item).is_dir()] - roots += [Path(item) for item in sorted(glob.glob("/usr/local/cuda*/lib64"))] + roots = _library_roots() library = next((root / soname for root in roots if (root / soname).exists()), None) if library is None: @@ -782,12 +813,9 @@ def _package_field(package: str, field: str) -> str | None: def _os_release_id() -> str | None: """The distribution this package manager belongs to, for the purl type.""" try: - for line in Path("/etc/os-release").read_text().splitlines(): - if line.startswith("ID="): - return line.partition("=")[2].strip().strip('"') or None + return platform.freedesktop_os_release().get("ID") or None except OSError: return None - return None def _package_supplier(package: str) -> str | None: @@ -796,18 +824,8 @@ def _package_supplier(package: str) -> str | None: The package name is not a supplier: `libbsd0` names the thing, not whoever provided it. dpkg records a maintainer, which is the answer. """ - try: - result = subprocess.run( - ["dpkg-query", "-W", "-f=${Maintainer}", package], - capture_output=True, - text=True, - check=False, - timeout=60, - ) - except (OSError, subprocess.SubprocessError): - return None - maintainer = result.stdout.strip() - if result.returncode != 0 or not maintainer: + maintainer = _package_field(package, "Maintainer") + if not maintainer: return None # "Name " is Debian's form; SPDX wants "Organization: Name (email)". match = re.match(r"^(?P.+?)\s*<(?P[^>]+)>$", maintainer) @@ -816,6 +834,12 @@ def _package_supplier(package: str) -> str | None: return f"Organization: {maintainer}" +def _library_roots() -> list[Path]: + """Where a pre-built library the build machine supplied can be found.""" + roots = [Path(item) for item in _SEARCH_ROOTS if Path(item).is_dir()] + return roots + [Path(item) for item in sorted(glob.glob("/usr/local/cuda*/lib64"))] + + def _license_near(library: Path) -> Path | None: """A toolchain that installs outside the package manager still ships its terms. @@ -825,9 +849,7 @@ def _license_near(library: Path) -> Path | None: current = library.parent for _ in range(3): for entry in sorted(current.iterdir()) if current.is_dir() else []: - if entry.is_file() and licensing._classify(entry.name): # noqa: SLF001 - return entry - if entry.is_file() and entry.name.upper().startswith("EULA"): + if entry.is_file() and licensing.classify(entry.name): return entry if current.parent == current: break @@ -847,7 +869,7 @@ def system_component(soname: str, origin: dict) -> Component: if license_path and Path(license_path).is_file(): text = Path(license_path).read_text(encoding="utf-8", errors="replace") evidence.append( - licensing._evidence(key, "build-host", license_path, text, "grant") # noqa: SLF001 + licensing.evidence_for(key, "build-host", license_path, text, "grant") ) # `key`, not `soname`: the key is what identifies this component everywhere # else, and two components reduced to the same label would mint one LicenseRef @@ -954,7 +976,7 @@ def _maybe_license(self, display: str, member: Member, data: bytes) -> None: text = data.decode("utf-8", "replace").strip().splitlines() self.archives[display]["version_text"] = text[0].strip() if text else "" return - if not licensing._classify(name): # noqa: SLF001 - same package + if not licensing.classify(name): return self.archives[display].setdefault("license_members", []).append((member, data)) diff --git a/scripts/sbom/sbom/document.py b/scripts/sbom/sbom/document.py index 8172492aaf..63cd9ed957 100644 --- a/scripts/sbom/sbom/document.py +++ b/scripts/sbom/sbom/document.py @@ -92,9 +92,7 @@ def _extracted_licenses( refs = licensing.license_refs( component.license_concluded, component.license_declared ) - # Same predicate as the build gate, the verifier and the report: a REUSE - # pool is a real text. Change one of those and change all of them. - grants = [item for item in component.evidence if item.kind in ("grant", "pool")] + grants = licensing.obligation_texts(component.evidence) if not refs or not grants: continue for ref in refs: @@ -266,7 +264,7 @@ def build_document( # A packaged licence text states its component's notice, or its own # author's -- WTFPL names the person who wrote WTFPL. Neither is a # notice this distribution makes about itself. - and not entry.name.startswith(f"{wheel.dist_info}/licenses/") + and not report.is_packaged_license(wheel.dist_info, entry.name) ): stated_copyright.add(attribution.copyright_text) diff --git a/scripts/sbom/sbom/evidence.py b/scripts/sbom/sbom/evidence.py index 06c9749a41..a1a44bca10 100644 --- a/scripts/sbom/sbom/evidence.py +++ b/scripts/sbom/sbom/evidence.py @@ -17,6 +17,7 @@ from .discovery import ( ARCHIVE_SUFFIXES, ArchiveIndex, + component_of, Ownership, git, BuildGraph, @@ -120,12 +121,12 @@ def _archive_components( name = name.removesuffix(suffix) evidence = [ - licensing._evidence( # noqa: SLF001 - same package + licensing.evidence_for( display, "archive-member", f"{display}!{member.path}", data.decode("utf-8", "replace"), - licensing._classify(Path(member.path).name) or "grant", # noqa: SLF001 + licensing.classify(Path(member.path).name) or "grant", ) for member, data in record.get("license_members", []) ] @@ -254,7 +255,7 @@ def discover(repo_root: Path, build_dir: Path) -> Discovery: continue match = source_files.by_hash.get(digest) if match: - owners.add(match.split("/", 1)[0].removesuffix("-src")) + owners.add(component_of(match)) if len(owners) == 1: display = f"{build_dir.name}/{root.relative_to(build_dir).as_posix()}" staged_component_dirs[display] = owners.pop() @@ -283,19 +284,7 @@ def discover(repo_root: Path, build_dir: Path) -> Discovery: for item in tracked if Path(item).suffix.lower() in _TAGGABLE_SOURCE } - for node in graph._nodes.values(): # noqa: SLF001 - same package - candidates.update( - source - for source in node.sources - if str(source).startswith(str(repo_root)) - and "/_deps/" not in source.as_posix() - ) - for include in node.includes: - text = include.as_posix() - if not text.startswith(str(repo_root)) or "/_deps/" in text: - continue - if include.is_dir() and not text.startswith(str(build_dir)): - candidates.update(item for item in include.rglob("*") if item.is_file()) + candidates.update(graph.repo_inputs(repo_root, build_dir)) vendored, owned_paths = discover_vendored( repo_root, candidates, project_authors(repo_root) diff --git a/scripts/sbom/sbom/inventory.py b/scripts/sbom/sbom/inventory.py index dc857b1303..da237fabc3 100644 --- a/scripts/sbom/sbom/inventory.py +++ b/scripts/sbom/sbom/inventory.py @@ -20,6 +20,7 @@ from .discovery import ( Component, EMPTY_SHA256, + component_of, resolve_system_library, resolve_system_library_by_build_id, sha256_file, @@ -238,7 +239,7 @@ def _trace( component_file = self.source_files.by_hash.get(candidate_digest) if component_file: - key = component_file.split("/", 1)[0].removesuffix("-src") + key = component_of(component_file) return Attribution( path=wheel_path, origin="copied", @@ -377,7 +378,7 @@ def _shared_content(self, digest: str): """ owners: dict[str, str] = {} for display in self.source_files.all_by_hash.get(digest, []): - owners[display.split("/", 1)[0].removesuffix("-src")] = display + owners[component_of(display)] = display for member in self.archives.all_by_hash.get(digest, []): owners[member.container] = f"{member.path} in {member.container}" # This repository holding the same bytes counts as a candidate too: a @@ -519,7 +520,7 @@ def _resolve_by_path( component_file = self.source_files.path_suffix_match(wheel_path) if component_file: - key = component_file.split("/", 1)[0].removesuffix("-src") + key = component_of(component_file) origin_digest = self.source_files.digest_of_display(component_file) # Reaching this branch means the content matched more than one place, # not that anything transformed the file. Where the input's bytes are diff --git a/scripts/sbom/sbom/licensing.py b/scripts/sbom/sbom/licensing.py index f822a74c1d..24624c9ca3 100644 --- a/scripts/sbom/sbom/licensing.py +++ b/scripts/sbom/sbom/licensing.py @@ -100,6 +100,22 @@ def as_json(self) -> dict: } +# What counts as a text that discharges the packaging obligation. A REUSE pool +# names no single expression for the component and is still a real text, so it +# counts. The build gate, the document, the report and the verifier all ask this +# question; they ask it here. +OBLIGATION_KINDS = frozenset({"grant", "pool"}) + + +def satisfies_obligation(kind: str) -> bool: + """For callers holding the JSON form, where there is no dataclass to filter.""" + return kind in OBLIGATION_KINDS + + +def obligation_texts(evidence) -> list[LicenseEvidence]: + return [item for item in evidence if satisfies_obligation(item.kind)] + + _DIGITS = re.compile(r"\d+") @@ -312,7 +328,7 @@ def _read(path: Path) -> str | None: return data.decode("utf-8", "replace") -def _evidence( +def evidence_for( component: str, origin: str, display: str, text: str, kind: str ) -> LicenseEvidence: matchable = kind == "grant" @@ -333,7 +349,7 @@ def _evidence( ) -def _classify(name: str) -> str | None: +def classify(name: str) -> str | None: if Path(name).suffix.lower() in _SKIP_SUFFIXES: return None if _GRANT_NAMES.search(name): @@ -351,7 +367,7 @@ def discover_in_tree( candidates: list[tuple[Path, str]] = [] for entry in sorted(root.iterdir()) if root.is_dir() else []: - if entry.is_file() and (kind := _classify(entry.name)): + if entry.is_file() and (kind := classify(entry.name)): candidates.append((entry, kind)) elif entry.is_dir() and entry.name.upper() in {"LICENSES", "LICENSE"}: candidates.extend( @@ -363,7 +379,7 @@ def discover_in_tree( if text is None: continue display = f"{display_prefix}/{path.relative_to(root).as_posix()}" - found.append(_evidence(component, "component-file", display, text, kind)) + found.append(evidence_for(component, "component-file", display, text, kind)) if not any(item.kind == "grant" for item in found): found.extend(_readme_fallback(component, root, display_prefix)) @@ -389,7 +405,9 @@ def _readme_fallback( # threshold and degrade to a LicenseRef. display = f"{display_prefix}/{entry.name}" return [ - _evidence(component, "readme-section", display, section.group(0), "grant") + evidence_for( + component, "readme-section", display, section.group(0), "grant" + ) ] return [] @@ -618,7 +636,7 @@ def canonical_evidence( continue seen.add(token) found.append( - _evidence( + evidence_for( component_key, "spdx-reference", f"{token}.txt", diff --git a/scripts/sbom/sbom/report.py b/scripts/sbom/sbom/report.py index eb3a6f1264..d0e259efa0 100644 --- a/scripts/sbom/sbom/report.py +++ b/scripts/sbom/sbom/report.py @@ -7,6 +7,7 @@ import re +from . import licensing from .inventory import Inventory _HOW_PROSE = { @@ -36,6 +37,35 @@ def _short(version: str) -> str: return version[:12] if _COMMIT.match(version) else version +def licenses_root(dist_info: str) -> str: + """Where a wheel carries licence texts. PEP 639 names them relative to this.""" + return f"{dist_info}/licenses/" + + +def notices_path(dist_info: str) -> str: + """The notice file that travels inside the wheel, by name.""" + return f"{licenses_root(dist_info)}THIRD-PARTY-NOTICES.md" + + +def is_packaged_license(dist_info: str, member: str) -> bool: + return member.startswith(licenses_root(dist_info)) + + +def is_third_party_license(dist_info: str, member: str) -> bool: + """A text read out of a component, as against this distribution's own.""" + return member.startswith(f"{licenses_root(dist_info)}third-party/") + + +def declared_license_path(dist_info: str, member: str) -> str: + """What METADATA calls a packaged licence file. + + PEP 639 states License-File relative to `.dist-info/licenses/`, which is + what setuptools already does for the project's own texts; declaring the + member path instead yields an entry that resolves to nothing. + """ + return member.removeprefix(licenses_root(dist_info)) + + def packaged_license_path(dist_info: str, component_key: str, evidence) -> str: """Where one component's license text lands inside the wheel. @@ -49,7 +79,7 @@ def packaged_license_path(dist_info: str, component_key: str, evidence) -> str: # scheme exists to avoid, and the loser was dropped while the document went # on naming the path it was dropped from. safe = "/".join(_SAFE_NAME.sub("_", part) for part in relative.split("/")) - return f"{dist_info}/licenses/third-party/{folder}/{safe}" + return f"{licenses_root(dist_info)}third-party/{folder}/{safe}" def _roles(inventory: Inventory, key: str) -> str: @@ -64,7 +94,7 @@ def notices_markdown(inventory: Inventory, dist_info: str, wheel_name: str) -> s "", f"Components redistributed in `{wheel_name}`, with the license text each", "obligation was read from. Texts are packaged under", - f"`{dist_info}/licenses/third-party/`.", + f"`{licenses_root(dist_info)}third-party/`.", "", ] for key, component in inventory.components_present.items(): @@ -97,7 +127,7 @@ def report(inventory: Inventory, evidence_doc: dict, wheel_name: str) -> dict: unidentified = [] for key, component in inventory.components_present.items(): - grants = [item for item in component.evidence if item.kind in ("grant", "pool")] + grants = licensing.obligation_texts(component.evidence) components.append( { "component": key, diff --git a/scripts/sbom/sbom/validate.py b/scripts/sbom/sbom/validate.py index 41ce41546a..737eba3036 100644 --- a/scripts/sbom/sbom/validate.py +++ b/scripts/sbom/sbom/validate.py @@ -15,7 +15,7 @@ import zipfile from pathlib import Path -from . import SPDX_VERSION, licensing +from . import SPDX_VERSION, licensing, report from . import document as document_module from . import wheelfile @@ -224,7 +224,7 @@ def _check_license_texts(wheel: wheelfile.WheelInfo, spdx: dict) -> list[str]: packaged = { digest for name, digest in by_name.items() - if name.startswith(f"{wheel.dist_info}/licenses/") + if report.is_packaged_license(wheel.dist_info, name) } if not packaged: return failures @@ -284,7 +284,7 @@ def _check_attribution(evidence_doc: dict, wheel: wheelfile.WheelInfo) -> list[s packaged = { entry.sha256 for entry in wheel.entries - if entry.name.startswith(f"{wheel.dist_info}/licenses/") + if report.is_packaged_license(wheel.dist_info, entry.name) } recorded: set[str] = set() @@ -301,13 +301,10 @@ def _check_attribution(evidence_doc: dict, wheel: wheelfile.WheelInfo) -> list[s # so comparing only the licence-granting ones called a NOTICE the build # plainly read a text it never did. recorded.update(item["sha256"] for item in component.get("evidence", [])) - # The same test the build gate applies: a REUSE pool is a real text even - # though it names no single expression, so publishing on one and then - # failing verification for want of a grant would contradict the gate. texts = [ item for item in component.get("evidence", []) - if item["kind"] in ("grant", "pool") + if licensing.satisfies_obligation(item["kind"]) ] if not texts: failures.append( @@ -429,7 +426,7 @@ def _refs_in(value) -> set[str]: def _check_licenses(wheel: wheelfile.WheelInfo, spdx: dict) -> list[str]: failures: list[str] = [] names = {item.name for item in wheel.entries} - notices = f"{wheel.dist_info}/licenses/THIRD-PARTY-NOTICES.md" + notices = report.notices_path(wheel.dist_info) if notices not in names: failures.append(f"{notices} is not packaged") diff --git a/scripts/sbom/sbom/wheelfile.py b/scripts/sbom/sbom/wheelfile.py index c4d866e940..ca64ab3def 100644 --- a/scripts/sbom/sbom/wheelfile.py +++ b/scripts/sbom/sbom/wheelfile.py @@ -100,25 +100,17 @@ def sbom_member(dist_info: str, wheel_name: str) -> str: return f"{dist_info}/sboms/{wheel_name}.spdx.json" -def record_hash(data: bytes) -> str: - """RECORD's hash field, as PEP 376 spells it.""" - return str(Hash.parse(f"sha256={_urlsafe_b64(data)}")) - - def record_hash_from_sha256(digest: str) -> str: - """The same field, from a digest already computed.""" + """RECORD's hash field, as PEP 376 spells it, from a digest already computed.""" encoded = ( base64.urlsafe_b64encode(bytes.fromhex(digest)).rstrip(b"=").decode("ascii") ) return str(Hash.parse(f"sha256={encoded}")) -def _urlsafe_b64(data: bytes) -> str: - return ( - base64.urlsafe_b64encode(hashlib.sha256(data).digest()) - .rstrip(b"=") - .decode("ascii") - ) +def record_hash(data: bytes) -> str: + """The same field, for bytes in hand.""" + return record_hash_from_sha256(hashlib.sha256(data).hexdigest()) def _dist_info(names: list[str]) -> str: diff --git a/tests/python/sbom/test_licensing.py b/tests/python/sbom/test_licensing.py index 1464a4ea3a..e411bdc967 100644 --- a/tests/python/sbom/test_licensing.py +++ b/tests/python/sbom/test_licensing.py @@ -120,7 +120,7 @@ def test_a_real_license_survives_surrounding_packaging_boilerplate(): def test_unmatched_grant_becomes_a_license_ref_not_a_guess(): evidence = [ - licensing._evidence( + licensing.evidence_for( "demo", "component-file", "demo/LICENSE", synth.PROPRIETARY_LICENSE, "grant" ) ] @@ -231,7 +231,7 @@ def test_a_component_carrying_two_licenses_combines_them(): exercised with more than one licence. """ evidence = [ - licensing._evidence( + licensing.evidence_for( "dual", "component-file", "dual/LICENSE", From 082fe20e1c105c24914f399a353def1a76d6a5b3 Mon Sep 17 00:00:00 2001 From: Andrei Aristarkhov Date: Thu, 1 Oct 2026 01:50:08 -0700 Subject: [PATCH 048/101] refactor(sbom): drop what was written twice, and what was never read `ArchiveIndex.by_build_id` was built and never read -- the one consumer uses `all_by_build_id` deliberately, because it must refuse to answer when several containers share a build-id, and `by_build_id` is exactly the first-wins answer the comment beside it forbids. `_from_repo_file` took a `source` it never used while both callers also folded it into the string they passed. Two SPDX "named but not opened" packages spelled out the same seven unasserted fields; the duplicate-detection scan was written three times as a `list.count` inside a comprehension over that same list, which is quadratic on a wheel's member count. `_record_for` matched its filename twice, and the document's licence-list version fell back to a literal -- a version the document was not built against is a false claim about every identification in it. The verifier now joins evidence to packages on the id the document assigns rather than on the display name: a source tree, a system library and an archive can share a name, and a name join drops all but one of them silently. Signed-off-by: Andrei Aristarkhov --- scripts/sbom/sbom/__init__.py | 5 +- scripts/sbom/sbom/build.py | 3 +- scripts/sbom/sbom/discovery.py | 10 +--- scripts/sbom/sbom/document.py | 87 +++++++++++++++++++--------------- scripts/sbom/sbom/inventory.py | 27 ++++------- scripts/sbom/sbom/licensing.py | 6 +-- scripts/sbom/sbom/validate.py | 56 +++++++++++----------- scripts/sbom/sbom/wheelfile.py | 3 +- 8 files changed, 95 insertions(+), 102 deletions(-) diff --git a/scripts/sbom/sbom/__init__.py b/scripts/sbom/sbom/__init__.py index 1c53e55367..8f5e4361e8 100644 --- a/scripts/sbom/sbom/__init__.py +++ b/scripts/sbom/sbom/__init__.py @@ -12,9 +12,8 @@ def stamped_now(): """The time to publish, honouring SOURCE_DATE_EPOCH. - Every artifact a release advertises uses this. Having only the SPDX document - honour the epoch made a rebuild comparable to a published wheel but not to - its published evidence, which is the chain the manifest binds. + Every artifact a release advertises uses this; the manifest binds them as one + chain, so they must agree on the clock. """ import os from datetime import datetime, timezone diff --git a/scripts/sbom/sbom/build.py b/scripts/sbom/sbom/build.py index 651759171c..d210518b48 100644 --- a/scripts/sbom/sbom/build.py +++ b/scripts/sbom/sbom/build.py @@ -10,6 +10,7 @@ import os import shutil import tempfile +from collections import Counter from dataclasses import replace from pathlib import Path @@ -257,7 +258,7 @@ def merge_manifests(paths: list[Path]) -> dict: def merge_records(wheels: list[dict]) -> dict: """The manifest for a set of wheels, stamped when the set was complete.""" names = [item["filename"] for item in wheels] - duplicates = sorted({name for name in names if names.count(name) > 1}) + duplicates = sorted(name for name, seen in Counter(names).items() if seen > 1) if duplicates: raise BuildError(f"the same wheel filename was published twice: {duplicates}") return { diff --git a/scripts/sbom/sbom/discovery.py b/scripts/sbom/sbom/discovery.py index b534d16756..4b02810a4c 100644 --- a/scripts/sbom/sbom/discovery.py +++ b/scripts/sbom/sbom/discovery.py @@ -937,7 +937,6 @@ def __init__(self) -> None: self.by_name: dict[str, list[Member]] = {} # Survives the RPATH/SONAME rewrite a repair tool applies on the way in, # so a patched wheel member still points at the archive it came from. - self.by_build_id: dict[str, Member] = {} self.all_by_build_id: dict[str, list[Member]] = {} self.archives: dict[str, dict] = {} @@ -959,7 +958,6 @@ def add_archive(self, path: Path, display: str) -> None: build_id = None if build_id: member = replace(member, build_id=build_id) - self.by_build_id.setdefault(build_id, member) self.all_by_build_id.setdefault(build_id, []).append(member) self.by_hash.setdefault(member.sha256, member) self.all_by_hash.setdefault(member.sha256, []).append(member) @@ -1082,12 +1080,8 @@ def digest_of(self, path: Path) -> str | None: def add_tree( self, root: Path, display_root: str, skip: tuple[Path, ...] = () ) -> None: - # Compare inside the tree being walked. `--build-dir` is free-form, and - # a relative root yields relative paths that match no absolute skip - # entry, so passing `--build-dir build` -- what the workflow passes -- - # indexed the staged tree and let a wheel member explain itself. A - # string prefix was wrong for a second reason: `_deps/foo-srcx` starts - # with `_deps/foo-src`. + # Compare skip entries as paths relative to `root`: `--build-dir` may be + # relative, and `_deps/foo-srcx` string-prefixes `_deps/foo-src`. inside = [] for item in skip: try: diff --git a/scripts/sbom/sbom/document.py b/scripts/sbom/sbom/document.py index 63cd9ed957..0cb5b29a37 100644 --- a/scripts/sbom/sbom/document.py +++ b/scripts/sbom/sbom/document.py @@ -28,6 +28,11 @@ def _spdx_id(prefix: str, value: str) -> str: return f"SPDXRef-{prefix}-{licensing.spdx_safe(value)}" +def component_package_id(key: str) -> str: + """The id this document gives a component. The verifier joins on it.""" + return _spdx_id("Package", key) + + def verification_code(sha1_digests: list[str]) -> str: """SPDX package verification code: SHA1 over the sorted file SHA1s.""" return hashlib.sha1("".join(sorted(sha1_digests)).encode("ascii")).hexdigest() # noqa: S324 @@ -59,7 +64,7 @@ def _attribution_texts(component: Component, dist_info: str) -> list[str]: def _component_package(component: Component, roles: set[str], dist_info: str) -> dict: package = { - "SPDXID": _spdx_id("Package", component.key), + "SPDXID": component_package_id(component.key), "name": component.name, "versionInfo": component.version, "supplier": component.supplier, @@ -179,6 +184,27 @@ def _share_identical_texts(entries: list[dict]) -> tuple[list[dict], dict[str, s _EXTRA_MARKER = re.compile(r"\bextra\s*==") +def _unanalyzed_package( + package_id: str, name: str, comment: str, external_refs: list[dict] | None = None +) -> dict: + """A package this document names but did not open: everything unasserted.""" + package = { + "SPDXID": package_id, + "name": name, + "versionInfo": NOASSERTION, + "supplier": NOASSERTION, + "downloadLocation": NOASSERTION, + "filesAnalyzed": False, + "licenseConcluded": NOASSERTION, + "licenseDeclared": NOASSERTION, + "copyrightText": NOASSERTION, + "comment": comment, + } + if external_refs: + package["externalRefs"] = external_refs + return package + + def _is_extra_gated(requirement: Requirement) -> bool: return requirement.marker is not None and bool( _EXTRA_MARKER.search(str(requirement.marker)) @@ -387,21 +413,12 @@ def build_document( for soname, consumers in inventory.external_runtime.items(): package_id = _spdx_id("Package-external", soname) packages.append( - { - "SPDXID": package_id, - "name": soname, - "versionInfo": NOASSERTION, - "supplier": NOASSERTION, - "downloadLocation": NOASSERTION, - "filesAnalyzed": False, - "licenseConcluded": NOASSERTION, - "licenseDeclared": NOASSERTION, - "copyrightText": NOASSERTION, - "comment": ( - "Supplied by the installation environment, not redistributed in " - f"this wheel. Required by: {', '.join(consumers)}." - ), - } + _unanalyzed_package( + package_id, + soname, + "Supplied by the installation environment, not redistributed in " + f"this wheel. Required by: {', '.join(consumers)}.", + ) ) relationships.append( { @@ -426,27 +443,18 @@ def build_document( ) extras = sorted({extra for item in group for extra in item.extras}) packages.append( - { - "SPDXID": package_id, - "name": name, - "versionInfo": NOASSERTION, - "supplier": NOASSERTION, - "downloadLocation": NOASSERTION, - "filesAnalyzed": False, - "licenseConcluded": NOASSERTION, - "licenseDeclared": NOASSERTION, - "copyrightText": NOASSERTION, - "externalRefs": _dependency_refs(name, group[0]), - "comment": ( - "Consumer requirement declared in wheel metadata: " - # " | ", not "; ": a requirement string contains its own - # semicolon before the marker. - + " | ".join(sorted(str(item) for item in group)) - + ". Resolved at install time; this document states no " - "license for it." - + (f" Requested with: {', '.join(extras)}." if extras else "") - ), - } + _unanalyzed_package( + package_id, + name, + "Consumer requirement declared in wheel metadata: " + # " | ", not "; ": a requirement string contains its own + # semicolon before the marker. + + " | ".join(sorted(str(item) for item in group)) + + ". Resolved at install time; this document states no license " + "for it." + + (f" Requested with: {', '.join(extras)}." if extras else ""), + _dependency_refs(name, group[0]), + ) ) # Optional only if every line that asks for it is gated on an extra. One # unconditional line makes the dependency unconditional. @@ -493,8 +501,11 @@ def build_document( f"Tool: {TOOL_NAME}-{TOOL_VERSION}", f"Organization: {project['supplier'].split(': ', 1)[-1]}", ], + # From the corpus this build identified against, never a default: a + # stated version the document was not built against is a false claim + # about every identification in it. "licenseListVersion": ".".join( - str(evidence_doc.get("license_list_version", "3.29")).split(".")[:2] + str(evidence_doc["license_list_version"]).split(".")[:2] ), "comment": ( "Contents inventory for one built wheel, discovered from the build " diff --git a/scripts/sbom/sbom/inventory.py b/scripts/sbom/sbom/inventory.py index da237fabc3..cdb9bea66b 100644 --- a/scripts/sbom/sbom/inventory.py +++ b/scripts/sbom/sbom/inventory.py @@ -262,9 +262,7 @@ def _trace( repo_file = self.repo_files.by_hash.get(candidate_digest) if repo_file: - return self._from_repo_file( - wheel_path, source, repo_file, f"{source}; ", True - ) + return self._from_repo_file(wheel_path, repo_file, f"{source}; ", True) build_file = self.build_files.by_hash.get(candidate_digest) if build_file: @@ -313,7 +311,7 @@ def _trace( return None def _from_repo_file( - self, wheel_path: str, source: str, repo_file: str, lead: str, same: bool + self, wheel_path: str, repo_file: str, lead: str, same: bool ) -> Attribution: """A file this repository holds -- and, where it is not ours, whose. @@ -516,7 +514,7 @@ def _resolve_by_path( if repo_file: same = self.repo_files.digest_of_display(repo_file) == digest lead = f"{source}; {'identical to' if same else 'derived from'} " - return self._from_repo_file(wheel_path, source, repo_file, lead, same) + return self._from_repo_file(wheel_path, repo_file, lead, same) component_file = self.source_files.path_suffix_match(wheel_path) if component_file: @@ -575,8 +573,7 @@ def build( if attribution is None: unattributed.append(entry.name) continue - attribution.spdx_tag = _spdx_tag(entry) - attribution.copyright_text = _copyright_text(entry) + attribution.spdx_tag, attribution.copyright_text = _head_facts(entry) attributions[entry.name] = attribution _resolve_shared_by_sibling(attributions) @@ -692,15 +689,9 @@ def _attribute( ) -def _copyright_text(entry: Entry) -> str | None: - """The notice a file states about itself, beside the identifier.""" - if Path(entry.name).suffix.lower() not in _TAGGABLE: - return None - return licensing.read_copyright(entry.head.decode("utf-8", "replace")) - - -def _spdx_tag(entry: Entry) -> str | None: - """REUSE tags are upstream stating the license of that exact file.""" +def _head_facts(entry: Entry) -> tuple[str | None, str | None]: + """The tag and notice a file states about itself: upstream, about that file.""" if Path(entry.name).suffix.lower() not in _TAGGABLE: - return None - return licensing.normalized_tag(entry.head.decode("utf-8", "replace")) + return None, None + head = entry.head.decode("utf-8", "replace") + return licensing.normalized_tag(head), licensing.read_copyright(head) diff --git a/scripts/sbom/sbom/licensing.py b/scripts/sbom/sbom/licensing.py index 24624c9ca3..591e66a6e2 100644 --- a/scripts/sbom/sbom/licensing.py +++ b/scripts/sbom/sbom/licensing.py @@ -259,10 +259,8 @@ def identify(text: str) -> list[tuple[str, float, float]]: file really can carry two licenses -- dual-licensed sources do -- so this returns every match that survives the family resolution below. - Every reference is compared. The thresholds do bound how long a reference - can be relative to the file, so matches could be skipped on length -- but - that ties correctness to the accept conditions never growing, and it fails - by quietly not identifying a license. Keep the comparison exhaustive. + Compare every reference; do not prune by length. Pruning ties correctness to + the accept conditions never growing, and fails by not identifying a license. """ candidate = _ngrams(_normalize(text)) if not candidate: diff --git a/scripts/sbom/sbom/validate.py b/scripts/sbom/sbom/validate.py index 737eba3036..74fefc3fcd 100644 --- a/scripts/sbom/sbom/validate.py +++ b/scripts/sbom/sbom/validate.py @@ -13,6 +13,7 @@ import json import re import zipfile +from collections import Counter from pathlib import Path from . import SPDX_VERSION, licensing, report @@ -45,21 +46,21 @@ def _record_for(manifest: dict, filename: str) -> dict: seen = [item for item in records if item.get("filename") == filename] if len(seen) > 1: raise ManifestError(f"{filename} is advertised {len(seen)} times") - for record in records: - if record.get("filename") == filename: - for key in ("sha256", "size", "sbom_in_wheel", "sidecars"): - if key not in record: - raise ManifestError(f"{filename}: manifest record has no {key!r}") - # Every sidecar, not just the ones a record happens to name. An - # absent entry is not an absent check: omitting `build_evidence` - # left `check-set` passing a wheel the evidence would have failed. - missing = sorted(set(_SIDECAR_KINDS) - set(record["sidecars"])) - if missing: - raise ManifestError( - f"{filename}: manifest record names no {', '.join(missing)} sidecar" - ) - return record - return {} + if not seen: + return {} + record = seen[0] + for key in ("sha256", "size", "sbom_in_wheel", "sidecars"): + if key not in record: + raise ManifestError(f"{filename}: manifest record has no {key!r}") + # Every sidecar, not just the ones a record happens to name. An absent entry + # is not an absent check: omitting `build_evidence` left `check-set` passing + # a wheel the evidence would have failed. + missing = sorted(set(_SIDECAR_KINDS) - set(record["sidecars"])) + if missing: + raise ManifestError( + f"{filename}: manifest record names no {', '.join(missing)} sidecar" + ) + return record _EVIDENCE_DIGEST = re.compile( @@ -121,7 +122,7 @@ def verify_wheel( identifiers = [spdx["SPDXID"]] identifiers += [item["SPDXID"] for item in spdx["packages"]] identifiers += [item["SPDXID"] for item in spdx["files"]] - duplicates = sorted({item for item in identifiers if identifiers.count(item) > 1}) + duplicates = sorted(item for item, seen in Counter(identifiers).items() if seen > 1) if duplicates: failures.append(f"duplicate SPDXIDs: {duplicates}") known = set(identifiers) @@ -335,9 +336,7 @@ def _document_agrees_with_evidence( evidence was produced before the wheel was published and states the digests independently, so where it is supplied it is the thing to compare against. """ - import zipfile as _zipfile - - with _zipfile.ZipFile(wheel.path) as archive: + with zipfile.ZipFile(wheel.path) as archive: name = wheelfile.sbom_member(wheel.dist_info, wheel.path.name) if name not in archive.namelist(): return [] @@ -358,12 +357,15 @@ def _document_agrees_with_evidence( # a proprietary EULA could be published as Apache-2.0, with the evidence # beside it still saying otherwise and every gate reporting OK. components = evidence_doc.get("components", {}) - by_name = { - component.get("name", key): (key, component) + # By the id the document assigns, not by display name: a source tree, a + # system library and an archive can all be called the same thing, and a name + # join silently drops all but one of them. + by_id = { + document_module.component_package_id(key): (key, component) for key, component in components.items() } for package in spdx.get("packages", []): - found = by_name.get(package["name"]) + found = by_id.get(package["SPDXID"]) if found is None: continue key, component = found @@ -390,11 +392,9 @@ def _document_agrees_with_evidence( # there is no single digest to compare and the join is left to the # packaged-path check. single = { - item["sha256"] + component["evidence"][0]["sha256"] for component in components.values() - for item in [component.get("evidence", [])] - if len(item) == 1 - for item in item + if len(component.get("evidence", [])) == 1 } for extracted in spdx.get("hasExtractedLicensingInfos", []): digest = hashlib.sha256( @@ -430,9 +430,7 @@ def _check_licenses(wheel: wheelfile.WheelInfo, spdx: dict) -> list[str]: if notices not in names: failures.append(f"{notices} is not packaged") - # Every field that can name one, through one parser. Two of these were not - # looked at, and packages were tokenised by hand while files went through the - # grammar -- two readings of one notation, disagreeing at the edges. + # Every field that can name one, through one parser. declared_refs = set() for item in spdx.get("files", []): for field in ("licenseConcluded", "licenseInfoInFiles"): diff --git a/scripts/sbom/sbom/wheelfile.py b/scripts/sbom/sbom/wheelfile.py index ca64ab3def..18eb0ba321 100644 --- a/scripts/sbom/sbom/wheelfile.py +++ b/scripts/sbom/sbom/wheelfile.py @@ -17,6 +17,7 @@ import os import time import zipfile +from collections import Counter from dataclasses import dataclass from email import message_from_bytes from email.generator import BytesGenerator @@ -140,7 +141,7 @@ def scan(wheel_path: Path, analyze_elf: bool = False) -> WheelInfo: # last of them, so the first is hashed by nothing, compared to RECORD by # nothing, and reported as uncovered by nothing -- while which one a # consumer extracts is up to their unzip. Refuse the archive. - repeated = sorted({item for item in names if names.count(item) > 1}) + repeated = sorted(item for item, seen in Counter(names).items() if seen > 1) if repeated: raise WheelError( f"more than one member is named {repeated}; a wheel member name " From acc1e04fa41b8b52dceea9fccb34b48617dece79 Mon Sep 17 00:00:00 2001 From: Andrei Aristarkhov Date: Thu, 1 Oct 2026 01:57:50 -0700 Subject: [PATCH 049/101] perf(sbom): read the build tree once per run, not once per wheel A run describes every wheel the build produced, and each one re-walked and re-hashed the whole build tree, re-decompressed every fetched archive, and re-ran the git and dpkg queries. The tree is the same tree; it is read once and shared now. What a wheel discovers for itself stays its own: `_from_build_host` adds the system libraries that wheel's members turned out to need, so the resolver holds a copy and the evidence sidecar is written from it. Sharing the dict would give the next wheel components it does not ship; writing the sidecar from the shared one instead left the document naming licence texts its evidence could not account for, and the wheel failed its own verification. The test builds two wheels from one `Discovery` and checks both directions. Smaller things on the same paths: `_walk` reads `is_dir`/`is_file` off the dirent instead of three stats per entry; `FileIndex` resolves the tree once rather than every file in it, and answers "what is under this display path" from the index instead of a second walk; the dependency closure is memoised per graph; artifact build-ids are read in the constructor and not again per member; the per-component file lists are indexed in one pass; archive members stream through the digest rather than being materialised whole, which sized peak memory by the largest file in an SDK tarball; and the verifier parses the embedded document once and hashes wheels and sidecars in chunks. Signed-off-by: Andrei Aristarkhov --- scripts/sbom/sbom/build.py | 14 ++++-- scripts/sbom/sbom/cli.py | 12 ++++- scripts/sbom/sbom/discovery.py | 78 ++++++++++++++++++++++++--------- scripts/sbom/sbom/evidence.py | 23 ++++++---- scripts/sbom/sbom/inventory.py | 35 ++++++++++----- scripts/sbom/sbom/validate.py | 42 ++++++++++++------ tests/python/sbom/test_gates.py | 57 ++++++++++++++++++++++++ 7 files changed, 204 insertions(+), 57 deletions(-) diff --git a/scripts/sbom/sbom/build.py b/scripts/sbom/sbom/build.py index d210518b48..765bb1d1b7 100644 --- a/scripts/sbom/sbom/build.py +++ b/scripts/sbom/sbom/build.py @@ -60,10 +60,16 @@ def build( out_dir: Path, license_data: Path | None = None, system_resolver=None, + discovery: evidence_module.Discovery | None = None, ) -> dict: - """Package evidence into `wheel_path` in place and write the sidecars.""" + """Package evidence into `wheel_path` in place and write the sidecars. + + `discovery` is the build tree read once. Reading it per wheel walks and + hashes the whole tree, and decompresses every fetched archive, again. + """ licensing.load_corpus(license_data or default_license_data(build_dir)) - discovery = evidence_module.discover(repo_root, build_dir) + if discovery is None: + discovery = evidence_module.discover(repo_root, build_dir) wheel = wheelfile.scan(wheel_path, analyze_elf=True) sbom_name = wheelfile.sbom_member(wheel.dist_info, wheel_path.name) if any(item.name == sbom_name for item in wheel.entries): @@ -165,7 +171,9 @@ def build( detail="the contents SBOM this collector embedded", ) - evidence_doc = evidence_module.document(discovery, wheel_path.name, inventory) + evidence_doc = evidence_module.document( + discovery, wheel_path.name, inventory, resolver.components + ) evidence_doc["ci"] = environment_summary() evidence_doc["system_libraries"] = resolver.system_libraries diff --git a/scripts/sbom/sbom/cli.py b/scripts/sbom/sbom/cli.py index 0b4e11c6e7..233b896ab1 100644 --- a/scripts/sbom/sbom/cli.py +++ b/scripts/sbom/sbom/cli.py @@ -64,10 +64,20 @@ def _cmd_build(args: argparse.Namespace) -> int: else build_module.default_license_data(build_dir) ) + licensing.load_corpus(license_data) + discovery = evidence_module.discover(repo_root, build_dir) + manifests: list[dict] = [] for wheel in sorted(Path(item) for item in args.wheel): manifests.append( - build_module.build(repo_root, build_dir, wheel, out_dir, license_data) + build_module.build( + repo_root, + build_dir, + wheel, + out_dir, + license_data, + discovery=discovery, + ) ) print(f"packaged evidence into {wheel.name}") diff --git a/scripts/sbom/sbom/discovery.py b/scripts/sbom/sbom/discovery.py index 4b02810a4c..e08d3cd4a1 100644 --- a/scripts/sbom/sbom/discovery.py +++ b/scripts/sbom/sbom/discovery.py @@ -21,6 +21,7 @@ import glob import hashlib import json +import os import platform import re import shutil @@ -61,10 +62,6 @@ EMPTY_SHA256 = hashlib.sha256(b"").hexdigest() -def sha256_bytes(data: bytes) -> str: - return hashlib.sha256(data).hexdigest() - - def component_of(display: str) -> str: """The component a `FileIndex` display path belongs to. @@ -84,21 +81,22 @@ def sha256_file(path: Path) -> str: def _walk(root: Path): + """Every regular file under `root`, never following a symlink.""" stack = [root] while stack: current = stack.pop() try: - entries = list(current.iterdir()) + entries = list(os.scandir(current)) except OSError: continue for entry in entries: if entry.is_symlink(): continue - if entry.is_dir(): + if entry.is_dir(follow_symlinks=False): if entry.name not in _SKIP_DIRS: - stack.append(entry) - elif entry.is_file(): - yield entry + stack.append(Path(entry.path)) + elif entry.is_file(follow_symlinks=False): + yield Path(entry.path) # ============================================================================== @@ -333,6 +331,7 @@ def __init__(self, build_dir: Path, config: str | None = None) -> None: self.artifacts: dict[Path, Artifact] = {} self._nodes: dict[str, _Node] = {} self._by_name: dict[str, list[Artifact]] = {} + self._closures: dict[str, tuple[_Node, ...]] = {} self._load(config) def _load(self, config: str | None) -> None: @@ -411,8 +410,15 @@ def repo_inputs(self, repo_root: Path, build_dir: Path) -> set[Path]: found.update(item for item in include.rglob("*") if item.is_file()) return found - def _closure(self, target_id: str) -> list[_Node]: - """The target and everything it depends on, transitively.""" + def _closure(self, target_id: str) -> tuple[_Node, ...]: + """The target and everything it depends on, transitively. + + Memoised per instance, not with `lru_cache`: a cache on the class would + hold every graph ever built alive for the life of the process. + """ + cached = self._closures.get(target_id) + if cached is not None: + return cached seen: set[str] = set() stack = [target_id] found: list[_Node] = [] @@ -424,7 +430,8 @@ def _closure(self, target_id: str) -> list[_Node]: node = self._nodes[current] found.append(node) stack.extend(node.dependencies) - return found + self._closures[target_id] = tuple(found) + return self._closures[target_id] def contributions( self, artifact: Artifact, ownership: Ownership | None = None @@ -941,13 +948,14 @@ def __init__(self) -> None: self.archives: dict[str, dict] = {} def add_archive(self, path: Path, display: str) -> None: - if path.stat().st_size > _MAX_ARCHIVE_BYTES: + size = path.stat().st_size + if size > _MAX_ARCHIVE_BYTES: return members = self._read_members(path, display) self.archives[display] = { "path": display, "sha256": sha256_file(path), - "size": path.stat().st_size, + "size": size, "member_count": len(members), } for member, data in members: @@ -988,17 +996,33 @@ def _read_members(path: Path, display: str) -> list[tuple[Member, bytes]]: """ collected: list[tuple[Member, bytes]] = [] - def take(name: str, data: bytes) -> None: - member = Member(display, name, sha256_bytes(data), len(data)) - keep = b"" if Path(name).parent.as_posix() not in {".", ""} else data - collected.append((member, keep)) + def take(name: str, handle) -> None: + # Only a root member's bytes are kept -- for its licence text or its + # ELF header -- so the rest stream through the digest. Reading every + # member whole sizes peak memory by the largest file in the archive. + at_root = Path(name).parent.as_posix() in {".", ""} + digest = hashlib.sha256() + size = 0 + kept = bytearray() + for chunk in iter(lambda: handle.read(1 << 20), b""): + digest.update(chunk) + size += len(chunk) + if at_root: + kept += chunk + collected.append( + ( + Member(display, name, digest.hexdigest(), size), + bytes(kept) if at_root else b"", + ) + ) try: if path.suffix == ".zip": with zipfile.ZipFile(path) as archive: for info in archive.infolist(): if not info.is_dir(): - take(info.filename, archive.read(info.filename)) + with archive.open(info) as handle: + take(info.filename, handle) else: with tarfile.open(path) as archive: for info in archive: @@ -1006,7 +1030,7 @@ def take(name: str, data: bytes) -> None: continue handle = archive.extractfile(info) if handle is not None: - take(info.name.removeprefix("./"), handle.read()) + take(info.name.removeprefix("./"), handle) except (OSError, tarfile.TarError, zipfile.BadZipFile) as error: # Only archives this build deliberately fetched are indexed, so one # that will not open is a broken download, not a stray file. Saying @@ -1105,6 +1129,9 @@ def add_paths(self, root: Path, display_root: str, paths) -> None: self._add(root, display_root, (root / item for item in paths)) def _add(self, root: Path, display_root: str, paths) -> None: + # One resolve for the tree, not one per file: `_walk` never crosses a + # symlink, so the resolved root plus the relative path is the real path. + resolved_root = root.resolve() for path in paths: if not path.is_file() or path.is_symlink(): continue @@ -1114,7 +1141,7 @@ def _add(self, root: Path, display_root: str, paths) -> None: digest = sha256_file(path) except OSError: continue - self._digests[str(path.resolve())] = digest + self._digests[str(resolved_root / relative)] = digest self.by_hash.setdefault(digest, display) self.all_by_hash.setdefault(digest, []).append(display) self._by_display[display] = digest @@ -1127,6 +1154,15 @@ def digest_of_display(self, display: str) -> str | None: """The digest this index recorded for one of its own display paths.""" return self._by_display.get(display) + def under(self, display_prefix: str): + """Every (display, digest) this index holds beneath a display path.""" + prefix = display_prefix.rstrip("/") + "/" + return ( + (display, digest) + for display, digest in self._by_display.items() + if display.startswith(prefix) + ) + def path_suffix_match(self, wheel_path: str) -> str | None: key = "/".join(wheel_path.rsplit("/", 2)[-2:]) matches = self.by_suffix.get(key, []) diff --git a/scripts/sbom/sbom/evidence.py b/scripts/sbom/sbom/evidence.py index a1a44bca10..39a7a7d981 100644 --- a/scripts/sbom/sbom/evidence.py +++ b/scripts/sbom/sbom/evidence.py @@ -30,7 +30,6 @@ _notice_from, tracked_files, EMPTY_SHA256, - _walk, ) # Suffixes whose files conventionally carry a REUSE header naming their holder. @@ -245,19 +244,20 @@ def discover(repo_root: Path, build_dir: Path) -> Discovery: for root in sorted(deps_dir.iterdir()): if not root.is_dir() or root.name.endswith("-src"): continue + display = f"{build_dir.name}/{root.relative_to(build_dir).as_posix()}" owners = set() - for path in _walk(root): - digest = build_files.digest_of(path) + # From the index rather than a second walk: `build_files` already hashed + # every file under the build tree. + for _, digest in build_files.under(display): # Zero bytes match every other empty file, so a CMake scaffolding # directory full of stamps would adopt whichever component happened # to ship an empty file. They are evidence of nothing here too. - if not digest or digest == EMPTY_SHA256: + if digest == EMPTY_SHA256: continue match = source_files.by_hash.get(digest) if match: owners.add(component_of(match)) if len(owners) == 1: - display = f"{build_dir.name}/{root.relative_to(build_dir).as_posix()}" staged_component_dirs[display] = owners.pop() # Third-party code checked in here rather than fetched: the candidates are @@ -310,8 +310,15 @@ def discover(repo_root: Path, build_dir: Path) -> Discovery: ) -def document(discovery: Discovery, wheel_name: str, inventory) -> dict: - """The build-evidence sidecar: resolved facts, not the raw indexes.""" +def document( + discovery: Discovery, wheel_name: str, inventory, components: dict[str, Component] +) -> dict: + """The build-evidence sidecar: resolved facts, not the raw indexes. + + `components` is what this wheel resolved against, which is the discovered set + plus the system libraries its own members turned out to need. One `Discovery` + serves every wheel in a run, so it cannot hold those. + """ repo_root = discovery.repo_root arch = { "x86_64": "amd64", @@ -386,7 +393,7 @@ def document(discovery: Discovery, wheel_name: str, inventory) -> dict: "evidence": [item.as_json() for item in component.evidence], "in_this_wheel": key in inventory.components_present, } - for key, component in sorted(discovery.components.items()) + for key, component in sorted(components.items()) }, "archives": { display: { diff --git a/scripts/sbom/sbom/inventory.py b/scripts/sbom/sbom/inventory.py index cdb9bea66b..67ee37c1ae 100644 --- a/scripts/sbom/sbom/inventory.py +++ b/scripts/sbom/sbom/inventory.py @@ -88,19 +88,30 @@ class Inventory: requires_dist: list[str] unattributed: list[str] = field(default_factory=list) absent_components: dict[str, str] = field(default_factory=dict) + _indexed: tuple[dict, dict] | None = field(default=None, repr=False) def files_of(self, key: str) -> list[str]: - return sorted( - path for path, item in self.attributions.items() if item.primary == key - ) + return self._by_component()[0].get(key, []) def files_sharing(self, key: str) -> list[str]: """Files naming this component as one candidate among several.""" - return sorted( - path - for path, item in self.attributions.items() - if item.primary is None and "shared-content" in item.components.get(key, ()) - ) + return self._by_component()[1].get(key, []) + + def _by_component(self) -> tuple[dict[str, list[str]], dict[str, list[str]]]: + """One pass over the members, not one per component per caller.""" + if self._indexed is None: + owned: dict[str, list[str]] = {} + shared: dict[str, list[str]] = {} + for path in sorted(self.attributions): + item = self.attributions[path] + if item.primary: + owned.setdefault(item.primary, []).append(path) + continue + for key, how in item.components.items(): + if "shared-content" in how: + shared.setdefault(key, []).append(path) + self._indexed = (owned, shared) + return self._indexed class Resolver: @@ -112,7 +123,9 @@ def __init__(self, discovery: Discovery, system_resolver=None) -> None: self.repo_root = discovery.repo_root self.build_dir = discovery.build_dir self.graph = discovery.graph - self.components = discovery.components + # A copy: `_from_build_host` adds the system libraries this wheel turned + # out to need, and one `Discovery` is shared by every wheel in a run. + self.components = dict(discovery.components) self.archives = discovery.archives self.repo_files = discovery.repo_files self.build_files = discovery.build_files @@ -121,6 +134,7 @@ def __init__(self, discovery: Discovery, system_resolver=None) -> None: self.staged_component_dirs = discovery.staged_component_dirs self._artifacts_by_hash: dict[str, list] = {} self._artifacts_by_build_id: dict[str, list] = {} + self._build_id_of: dict[Path, str | None] = {} for artifact in self.graph.artifacts.values(): if artifact.output.is_file(): # build_files already hashed everything under the build tree. @@ -129,6 +143,7 @@ def __init__(self, discovery: Discovery, system_resolver=None) -> None: ) self._artifacts_by_hash.setdefault(digest, []).append(artifact) build_id = elf.read_build_id(artifact.output) + self._build_id_of[artifact.output] = build_id if build_id: self._artifacts_by_build_id.setdefault(build_id, []).append( artifact @@ -497,7 +512,7 @@ def _resolve_by_path( artifacts = [ artifact for artifact in self.graph.by_name(name) - if not _build_ids_disagree(build_id, elf.read_build_id(artifact.output)) + if not _build_ids_disagree(build_id, self._build_id_of.get(artifact.output)) ] if len(artifacts) == 1: return self._from_artifact( diff --git a/scripts/sbom/sbom/validate.py b/scripts/sbom/sbom/validate.py index 74fefc3fcd..e0259fa4fd 100644 --- a/scripts/sbom/sbom/validate.py +++ b/scripts/sbom/sbom/validate.py @@ -17,6 +17,7 @@ from pathlib import Path from . import SPDX_VERSION, licensing, report +from .discovery import sha256_file from . import document as document_module from . import wheelfile @@ -97,9 +98,15 @@ def _embedded_sbom( def verify_wheel( - wheel_path: Path, scanned: wheelfile.WheelInfo | None = None + wheel_path: Path, + scanned: wheelfile.WheelInfo | None = None, + carried: list[dict] | None = None, ) -> list[str]: - """Verify a wheel against the SBOM it carries. No repository required.""" + """Verify a wheel against the SBOM it carries. No repository required. + + `carried` collects the parsed document for callers that go on to check it + against evidence, so the wheel is opened and the document parsed once. + """ failures: list[str] = [] wheel = scanned or wheelfile.scan(wheel_path) @@ -108,6 +115,8 @@ def verify_wheel( sbom_name, spdx = _embedded_sbom(archive, wheel.dist_info, wheel_path.name) except (LookupError, json.JSONDecodeError) as error: return [f"embedded SBOM: {error}"] + if carried is not None: + carried.append(spdx) for key in _REQUIRED_KEYS: if key not in spdx: @@ -271,7 +280,9 @@ def _check_record(wheel_path: Path, wheel: wheelfile.WheelInfo) -> list[str]: return failures -def _check_attribution(evidence_doc: dict, wheel: wheelfile.WheelInfo) -> list[str]: +def _check_attribution( + evidence_doc: dict, wheel: wheelfile.WheelInfo, spdx: dict | None +) -> list[str]: """Every wheel member must name where it came from, and every shipped component must have had a license text to read.""" stated = evidence_doc.get("wheel") @@ -322,12 +333,15 @@ def _check_attribution(evidence_doc: dict, wheel: wheelfile.WheelInfo) -> list[s f"{key} is redistributed but the license text recorded for it is " "not packaged in this wheel" ) - failures.extend(_document_agrees_with_evidence(wheel, recorded, evidence_doc)) + failures.extend(_document_agrees_with_evidence(wheel, recorded, evidence_doc, spdx)) return failures def _document_agrees_with_evidence( - wheel: wheelfile.WheelInfo, recorded: set[str], evidence_doc: dict + wheel: wheelfile.WheelInfo, + recorded: set[str], + evidence_doc: dict, + spdx: dict | None, ) -> list[str]: """The texts the document names must be the ones the build actually read. @@ -336,11 +350,8 @@ def _document_agrees_with_evidence( evidence was produced before the wheel was published and states the digests independently, so where it is supplied it is the thing to compare against. """ - with zipfile.ZipFile(wheel.path) as archive: - name = wheelfile.sbom_member(wheel.dist_info, wheel.path.name) - if name not in archive.namelist(): - return [] - spdx = json.loads(archive.read(name)) + if spdx is None: + return [] failures = [] for package in spdx.get("packages", []): @@ -471,11 +482,14 @@ def check( release has the whole set. Each input adds checks; none replaces the others. """ scanned = wheelfile.scan(wheel) - failures = verify_wheel(wheel, scanned) + carried: list[dict] = [] + failures = verify_wheel(wheel, scanned, carried) if build_evidence is not None: evidence_doc = json.loads(build_evidence.read_text(encoding="utf-8")) - failures.extend(_check_attribution(evidence_doc, scanned)) + failures.extend( + _check_attribution(evidence_doc, scanned, carried[0] if carried else None) + ) if manifest is not None: failures.extend(_check_manifest(manifest, wheel, evidence_dir)) @@ -493,7 +507,7 @@ def _check_manifest( if not record: return [f"{manifest_path.name} has no entry for {wheel.name}"] - digest = hashlib.sha256(wheel.read_bytes()).hexdigest() + digest = sha256_file(wheel) if record["sha256"] != digest: failures.append( "the wheel was modified after its digest was bound: manifest says " @@ -505,7 +519,7 @@ def _check_manifest( path = beside / sidecar["filename"] if not path.is_file(): failures.append(f"sidecar {kind} is missing: {sidecar['filename']}") - elif hashlib.sha256(path.read_bytes()).hexdigest() != sidecar["sha256"]: + elif sha256_file(path) != sidecar["sha256"]: failures.append(f"sidecar {kind} does not match its recorded digest") published = beside / record["sidecars"]["spdx"]["filename"] diff --git a/tests/python/sbom/test_gates.py b/tests/python/sbom/test_gates.py index 45d2237140..d7c9d457d8 100644 --- a/tests/python/sbom/test_gates.py +++ b/tests/python/sbom/test_gates.py @@ -448,3 +448,60 @@ def test_two_members_under_one_name_are_refused(built): with pytest.raises(wheelfile.WheelError, match="more than one member"): validate_module.check(smuggled) + + +def test_one_discovery_serves_every_wheel_without_bleeding_between_them( + workspace, tmp_path +): + """The build tree is read once for a run; a wheel's own findings are its own. + + `_from_build_host` adds the system libraries a wheel turned out to need. If + those land on the shared `Discovery`, the next wheel's evidence inherits + components it does not ship; if they land nowhere the evidence reaches, the + document names licence texts the evidence cannot account for and the wheel + fails its own verification. + """ + from sbom import evidence as evidence_module + + copyright_file = tmp_path / "copyright" + copyright_file.write_text(synth.spdx_text("Zlib"), encoding="utf-8") + payload = synth.wheel_payload(workspace) + payload["isaaccapture.libs/libz-a1b2c3.so"] = synth.elf_shared_object( + "libz.so.1", ("libc.so.6",) + ) + synth.write_wheel(workspace.wheel, payload) + + plain = workspace.wheel.parent / "plain-1.0-py3-none-any.whl" + synth.write_wheel(plain, synth.wheel_payload(workspace)) + + licensing.load_corpus(workspace.build / "_deps" / "license-list-data-src" / "json") + discovery = evidence_module.discover(workspace.root, workspace.build) + resolver = lambda soname: { # noqa: E731 + "soname": soname, + "resolved_path": "/usr/lib/libz.so.1.3", + "package": "zlib1g", + "supplier": "Organization: Ubuntu Developers (ubuntu-devel@example.com)", + "copyright": str(copyright_file), + } + for wheel in (workspace.wheel, plain): + build_module.build( + workspace.root, + workspace.build, + wheel, + workspace.root / "sbom", + system_resolver=resolver, + discovery=discovery, + ) + + def components(wheel_name): + stem = wheel_name.removesuffix(".whl") + sidecar = workspace.root / "sbom" / f"{stem}.build-evidence.json" + return json.loads(sidecar.read_text())["components"] + + assert "system:libz.so.1" in components(WHEEL_NAME), ( + "the wheel's own evidence must account for what its members needed" + ) + assert "system:libz.so.1" not in components(plain.name), ( + "a wheel that ships no such library must not inherit it" + ) + assert validate_module.check(workspace.wheel) == [] From 27f3d6c717e27d163d1bd27170ca0fb5fe58bdc2 Mon Sep 17 00:00:00 2001 From: Andrei Aristarkhov Date: Thu, 1 Oct 2026 02:03:20 -0700 Subject: [PATCH 050/101] build(sbom): point a requirement this build also made at its own document The transition wheel requires the wheel built beside it, and its document said nothing about it: name, and NOASSERTION for the version, the supplier and the licence. That reads as something an installer resolves from PyPI, for a distribution this same build produced and described in full. It is now an SPDX `externalDocumentRefs` entry -- the sibling's namespace and the SHA1 of its document -- and the dependency relationship points straight at `SPDXRef-Package-wheel` inside it. No local stub beside it: a stub would restate the version, supplier and licence from the same source, which is the duplication that made the rows worth looking at in the first place. Wheels are ordered by their requirements rather than by filename, because a document that cites another's digest has to be written after it. The verifier checks the new shape: an element written `DocumentRef-x:SPDXRef-y` must name a document this one declares, and every declared document must carry a SHA1 to check it by. Signed-off-by: Andrei Aristarkhov --- scripts/sbom/sbom/build.py | 69 +++++++++++++++++++++++ scripts/sbom/sbom/cli.py | 27 +++++---- scripts/sbom/sbom/document.py | 84 ++++++++++++++++++++++------ scripts/sbom/sbom/validate.py | 26 ++++++++- tests/python/sbom/test_wheel_sbom.py | 79 ++++++++++++++++++++++++++ 5 files changed, 255 insertions(+), 30 deletions(-) diff --git a/scripts/sbom/sbom/build.py b/scripts/sbom/sbom/build.py index 765bb1d1b7..3caeca70cb 100644 --- a/scripts/sbom/sbom/build.py +++ b/scripts/sbom/sbom/build.py @@ -21,6 +21,9 @@ from . import inventory as inventory_module from . import report as report_module from . import wheelfile +from packaging.requirements import Requirement +from packaging.utils import canonicalize_name + from .discovery import project_supplier, sha256_file from .inventory import Attribution from .wheelfile import Entry @@ -53,6 +56,68 @@ def default_license_data(build_dir: Path) -> Path: return build_dir / "_deps" / "license-list-data-src" / "json" +def in_dependency_order(wheels: list[Path]) -> list[Path]: + """Wheels of one run, each after any sibling it requires. + + A document that references another cites its digest, so the referenced one + has to exist first. Order is derived from the requirements rather than from + the filenames, which only happen to sort the right way. + """ + metadata = {} + for wheel in wheels: + info = wheelfile.scan(wheel) + raw, _ = wheelfile.read_metadata(wheel, info.dist_info) + parsed = wheelfile.parse_metadata(raw) + metadata[wheel] = ( + canonicalize_name(parsed["name"]), + { + canonicalize_name(Requirement(item).name) + for item in parsed.get("requires_dist") or [] + }, + ) + + ordered: list[Path] = [] + placed: set[str] = set() + remaining = list(wheels) + while remaining: + ready = [ + wheel + for wheel in remaining + if not ( + {name for name, _ in metadata.values()} + & metadata[wheel][1] - placed - {metadata[wheel][0]} + ) + ] + # A cycle between two wheels of one run cannot be ordered; keep the + # caller's order rather than refusing to describe either. + batch = ready or remaining + for wheel in batch: + ordered.append(wheel) + placed.add(metadata[wheel][0]) + remaining = [item for item in remaining if item not in batch] + return ordered + + +def _describes(spdx: dict, metadata_raw: bytes, sbom_filename: str) -> dict: + """What another wheel's document needs to cite this one, SPDX 2.3 style.""" + metadata = wheelfile.parse_metadata(metadata_raw) + body = json.dumps(spdx, indent=2, sort_keys=True).encode("utf-8") + b"\n" + wheel_package = next( + item + for item in spdx["packages"] + if item["SPDXID"] == document_module.WHEEL_PACKAGE_ID + ) + return { + "name": metadata["name"], + "version": metadata.get("version") or "NOASSERTION", + "namespace": spdx["documentNamespace"], + "sha1": hashlib.sha1(body).hexdigest(), # noqa: S324 + "filename": sbom_filename, + "license_declared": wheel_package["licenseDeclared"], + "supplier": wheel_package["supplier"], + } + + def build( repo_root: Path, build_dir: Path, @@ -61,6 +126,7 @@ def build( license_data: Path | None = None, system_resolver=None, discovery: evidence_module.Discovery | None = None, + siblings: dict[str, dict] | None = None, ) -> dict: """Package evidence into `wheel_path` in place and write the sidecars. @@ -183,6 +249,7 @@ def build( projected_wheel, inventory, excluded, + siblings, ) spdx_bytes = json.dumps(spdx, indent=2, sort_keys=True).encode("utf-8") + b"\n" additions[sbom_name] = spdx_bytes @@ -233,6 +300,8 @@ def build( "sha256": sha256_file(wheel_path), "size": wheel_path.stat().st_size, "sbom_in_wheel": sbom_name, + # What a sibling wheel's document cites to reference this one. + "describes": _describes(spdx, metadata_raw, sidecars["spdx"].name), "components_without_license_evidence": payload[ "components_without_license_evidence" ], diff --git a/scripts/sbom/sbom/cli.py b/scripts/sbom/sbom/cli.py index 233b896ab1..577161fae8 100644 --- a/scripts/sbom/sbom/cli.py +++ b/scripts/sbom/sbom/cli.py @@ -11,6 +11,8 @@ import sys from pathlib import Path +from packaging.utils import canonicalize_name + from . import TOOL_NAME, TOOL_VERSION from . import build as build_module from . import discovery @@ -68,17 +70,22 @@ def _cmd_build(args: argparse.Namespace) -> int: discovery = evidence_module.discover(repo_root, build_dir) manifests: list[dict] = [] - for wheel in sorted(Path(item) for item in args.wheel): - manifests.append( - build_module.build( - repo_root, - build_dir, - wheel, - out_dir, - license_data, - discovery=discovery, - ) + siblings: dict[str, dict] = {} + for wheel in build_module.in_dependency_order( + sorted(Path(item) for item in args.wheel) + ): + manifest = build_module.build( + repo_root, + build_dir, + wheel, + out_dir, + license_data, + discovery=discovery, + siblings=siblings, ) + manifests.append(manifest) + described = manifest["wheels"][0]["describes"] + siblings[canonicalize_name(described["name"])] = described print(f"packaged evidence into {wheel.name}") # Through the same merge as a release: inheriting the first wheel's manifest diff --git a/scripts/sbom/sbom/document.py b/scripts/sbom/sbom/document.py index 0cb5b29a37..a8debc3d75 100644 --- a/scripts/sbom/sbom/document.py +++ b/scripts/sbom/sbom/document.py @@ -10,6 +10,7 @@ from packageurl import PackageURL from packaging.requirements import Requirement +from packaging.utils import canonicalize_name from . import SPDX_VERSION, TOOL_NAME, TOOL_VERSION, licensing, report, stamped_now from .discovery import Component @@ -24,6 +25,10 @@ _AMBIGUOUS_HOWS = {"shared-content"} +# The package every document describes: the wheel it is about. +WHEEL_PACKAGE_ID = "SPDXRef-Package-wheel" + + def _spdx_id(prefix: str, value: str) -> str: return f"SPDXRef-{prefix}-{licensing.spdx_safe(value)}" @@ -211,7 +216,9 @@ def _is_extra_gated(requirement: Requirement) -> bool: ) -def _dependency_refs(name: str, requirement: Requirement) -> list[dict]: +def _dependency_refs( + name: str, requirement: Requirement, version: str | None = None +) -> list[dict]: """purl for the package, and the constraint as a range rather than prose. purl carries one exact version, which a requirement does not have. VERS (the @@ -222,7 +229,9 @@ def _dependency_refs(name: str, requirement: Requirement) -> list[dict]: { "referenceCategory": "PACKAGE-MANAGER", "referenceType": "purl", - "referenceLocator": PackageURL(type="pypi", name=name).to_string(), + "referenceLocator": PackageURL( + type="pypi", name=name, version=version + ).to_string(), } ] constraints = sorted( @@ -245,10 +254,17 @@ def build_document( wheel: WheelInfo, inventory: Inventory, excluded_files: list[str], + siblings: dict[str, dict] | None = None, ) -> dict: - """Assemble the document. Callers validate it; this only renders.""" + """Assemble the document. Callers validate it; this only renders. + + `siblings` are the other distributions of this same run, by canonical name. + A requirement naming one is not an install-time unknown: the build made it, + and its own document describes it. + """ files: list[dict] = [] file_ids: dict[str, str] = {} + external_documents: list[dict] = [] sha1_digests: list[str] = [] # What the files themselves state, for the package-level roll-up below. @@ -307,7 +323,7 @@ def build_document( files.append(record) code = verification_code(sha1_digests) - wheel_id = "SPDXRef-Package-wheel" + wheel_id = WHEEL_PACKAGE_ID build = evidence_doc.get("build", {}) wheel_package = { @@ -442,36 +458,67 @@ def build_document( f"{name}-{hashlib.sha256(str(sorted(map(str, group))).encode()).hexdigest()[:8]}", ) extras = sorted({extra for item in group for extra in item.extras}) - packages.append( - _unanalyzed_package( - package_id, - name, - "Consumer requirement declared in wheel metadata: " - # " | ", not "; ": a requirement string contains its own - # semicolon before the marker. - + " | ".join(sorted(str(item) for item in group)) - + ". Resolved at install time; this document states no license " - "for it." - + (f" Requested with: {', '.join(extras)}." if extras else ""), - _dependency_refs(name, group[0]), + declared = ( + "Consumer requirement declared in wheel metadata: " + # " | ", not "; ": a requirement string contains its own semicolon + # before the marker. + + " | ".join(sorted(str(item) for item in group)) + + "." + + (f" Requested with: {', '.join(extras)}." if extras else "") + ) + sibling = (siblings or {}).get(canonicalize_name(name)) + if sibling is None: + element = package_id + packages.append( + _unanalyzed_package( + package_id, + name, + f"{declared} Resolved at install time; this document states " + "no license for it.", + _dependency_refs(name, group[0]), + ) + ) + else: + # Built by this same run, so it is not an install-time unknown: the + # dependency points straight at the package in that wheel's own + # document. A local stub beside it would state the version, supplier + # and licence a second time, from the same source. + element = f"DocumentRef-{licensing.spdx_safe(sibling['filename'])}" + external_documents.append( + { + "externalDocumentId": element, + "spdxDocument": sibling["namespace"], + "checksum": { + "algorithm": "SHA1", + "checksumValue": sibling["sha1"], + }, + } ) + element = f"{element}:{WHEEL_PACKAGE_ID}" + + comment = declared + ( + f" Built by this same build as {sibling['name']} {sibling['version']}." + if sibling + else "" ) # Optional only if every line that asks for it is gated on an extra. One # unconditional line makes the dependency unconditional. if all(_is_extra_gated(item) for item in group): relationships.append( { - "spdxElementId": package_id, + "spdxElementId": element, "relatedSpdxElement": wheel_id, "relationshipType": "OPTIONAL_DEPENDENCY_OF", + "comment": comment, } ) else: relationships.append( { "spdxElementId": wheel_id, - "relatedSpdxElement": package_id, + "relatedSpdxElement": element, "relationshipType": "DEPENDS_ON", + "comment": comment, } ) @@ -517,4 +564,5 @@ def build_document( "files": files, "relationships": relationships, "hasExtractedLicensingInfos": extracted, + **({"externalDocumentRefs": external_documents} if external_documents else {}), } diff --git a/scripts/sbom/sbom/validate.py b/scripts/sbom/sbom/validate.py index e0259fa4fd..ea06f14fdc 100644 --- a/scripts/sbom/sbom/validate.py +++ b/scripts/sbom/sbom/validate.py @@ -135,13 +135,35 @@ def verify_wheel( if duplicates: failures.append(f"duplicate SPDXIDs: {duplicates}") known = set(identifiers) + # An element in another document is written `DocumentRef-x:SPDXRef-y`, and + # the prefix has to be one this document declares -- otherwise it names a + # document no reader can fetch or check. + declared_documents = { + item["externalDocumentId"] for item in spdx.get("externalDocumentRefs", []) + } for relationship in spdx["relationships"]: for side in ("spdxElementId", "relatedSpdxElement"): - if relationship[side] not in known: + element = relationship[side] + prefix, _, _ = element.partition(":") + if prefix.startswith("DocumentRef-"): + if prefix not in declared_documents: + failures.append( + f"relationship {relationship['relationshipType']} references " + f"{element!r}, and no externalDocumentRef declares {prefix!r}" + ) + continue + if element not in known: failures.append( f"relationship {relationship['relationshipType']} references unknown " - f"element {relationship[side]!r}" + f"element {element!r}" ) + for item in spdx.get("externalDocumentRefs", []): + checksum = item.get("checksum") or {} + if checksum.get("algorithm") != "SHA1" or not checksum.get("checksumValue"): + failures.append( + f"{item.get('externalDocumentId')!r} names a document with no SHA1 " + "to check it by" + ) wheel_package = next( ( diff --git a/tests/python/sbom/test_wheel_sbom.py b/tests/python/sbom/test_wheel_sbom.py index 884cf5da9a..92084c8a04 100644 --- a/tests/python/sbom/test_wheel_sbom.py +++ b/tests/python/sbom/test_wheel_sbom.py @@ -5,6 +5,7 @@ from __future__ import annotations +import hashlib import json import re import zipfile @@ -293,3 +294,81 @@ def test_components_sharing_a_licence_text_share_one_entry(workspace, license_da } defined = {item["licenseId"] for item in spdx["hasExtractedLicensingInfos"]} assert used <= defined, f"undefined after sharing: {sorted(used - defined)}" + + +def test_a_requirement_this_run_also_built_points_at_its_own_document( + workspace, license_data, tmp_path +): + """A transition wheel requires the wheel beside it; that is not an unknown. + + Left as an ordinary requirement it reads as something an installer resolves, + with the version, supplier and licence unasserted -- for a distribution this + same build produced and described in full. + """ + from sbom import cli + + synth.write_wheel(workspace.wheel, synth.wheel_payload(workspace)) + companion = workspace.wheel.parent / "isaacteleop-0.4.0-py3-none-any.whl" + synth.write_wheel( + companion, + { + "isaacteleop-0.4.0.dist-info/METADATA": "\n".join( + [ + "Metadata-Version: 2.4", + "Name: isaacteleop", + "Version: 0.4.0", + "License-Expression: Apache-2.0", + "Requires-Dist: isaaccapture>=0.4.0", + "", + ] + ).encode(), + "isaacteleop-0.4.0.dist-info/WHEEL": ( + b"Wheel-Version: 1.0\nGenerator: setuptools\nRoot-Is-Purelib: true\n" + ), + }, + ) + + out = workspace.root / "sbom" + # Through the CLI: it is what orders the wheels, and a document may only + # cite one that already exists. + assert ( + cli.main( + [ + "--repo-root", + str(workspace.root), + "build", + "--build-dir", + str(workspace.build), + "--license-data", + str(license_data), + "--out-dir", + str(out), + "--wheel", + str(companion), + "--wheel", + str(workspace.wheel), + ] + ) + == 0 + ) + + spdx = json.loads((out / "isaacteleop-0.4.0-py3-none-any.spdx.json").read_text()) + assert not [ + package + for package in spdx["packages"] + if package["SPDXID"].startswith("SPDXRef-Package-pypi-isaaccapture") + ], "the sibling is not restated as an install-time unknown" + + ref = spdx["externalDocumentRefs"][0] + sibling = (out / f"{WHEEL_NAME.removesuffix('.whl')}.spdx.json").read_bytes() + assert ref["spdxDocument"] == json.loads(sibling)["documentNamespace"] + assert ref["checksum"] == { + "algorithm": "SHA1", + "checksumValue": hashlib.sha1(sibling).hexdigest(), # noqa: S324 + } + assert any( + item["relatedSpdxElement"] + == f"{ref['externalDocumentId']}:SPDXRef-Package-wheel" + for item in spdx["relationships"] + ) + assert validate_module.check(companion) == [] From 21afc76d3c1d1b09d7c2d43dfdb24b57b56e1ccb Mon Sep 17 00:00:00 2001 From: Andrei Aristarkhov Date: Thu, 1 Oct 2026 02:29:35 -0700 Subject: [PATCH 051/101] build(sbom): close the gaps a code review found, and one its fix opened A library nested inside an SDK archive kept none of its bytes, so nothing read its build-id -- the evidence that survives a repair tool rewriting SONAME and RPATH. In this build that cost two members (both `plugins/libNvcfBlasUpload.so`; CloudXR keeps the rest at the archive root), but the rule it defeated is the one that refuses to name an archive when two disagree. A bounded head is kept for any member that starts with the ELF magic, which is enough: the note sits with the program headers. Alongside: `FileApiError` reaches the CLI's handler, so a tree with no file API reply prints its instruction instead of a traceback; a hand-written `SPDX-License-Identifier` that is not an expression -- "Public domain or MIT License" -- yields no evidence rather than raising out of the vendored scan; an unreadable ELF in a wheel is a collector error; a manifest naming an SBOM member the wheel lacks is reported, not a `KeyError`; archive directories are skipped by path segment, so `dist` no longer skips `distribution`; and ordering the wheels of a release no longer hashes every member of each to learn a name the central directory states. The review also found that holding a LicenseRef's body to a single file's digest failed any component stating its terms across several files, and loosened the check to components with exactly one. That left the shared ids -- which is what the proprietary SDK terms travel under -- unchecked: their body could be replaced wholesale and both gates passed. The build now states the digest of the terms as the document carries them, joined once in `licensing.verbatim_terms`, and the verifier holds every entry to it. Both cases have a test, each verified against the version that failed it. Signed-off-by: Andrei Aristarkhov --- scripts/sbom/sbom/build.py | 4 +- scripts/sbom/sbom/cli.py | 4 ++ scripts/sbom/sbom/discovery.py | 38 +++++++------ scripts/sbom/sbom/document.py | 7 ++- scripts/sbom/sbom/elf.py | 16 ++++++ scripts/sbom/sbom/evidence.py | 8 +++ scripts/sbom/sbom/licensing.py | 23 ++++++-- scripts/sbom/sbom/validate.py | 40 +++++++++----- scripts/sbom/sbom/wheelfile.py | 20 ++++++- tests/python/sbom/test_wheel_sbom.py | 80 ++++++++++++++++++++++++++++ 10 files changed, 202 insertions(+), 38 deletions(-) diff --git a/scripts/sbom/sbom/build.py b/scripts/sbom/sbom/build.py index 3caeca70cb..9fd1210be9 100644 --- a/scripts/sbom/sbom/build.py +++ b/scripts/sbom/sbom/build.py @@ -65,9 +65,7 @@ def in_dependency_order(wheels: list[Path]) -> list[Path]: """ metadata = {} for wheel in wheels: - info = wheelfile.scan(wheel) - raw, _ = wheelfile.read_metadata(wheel, info.dist_info) - parsed = wheelfile.parse_metadata(raw) + _, parsed = wheelfile.read_metadata(wheel, wheelfile.dist_info_of(wheel)) metadata[wheel] = ( canonicalize_name(parsed["name"]), { diff --git a/scripts/sbom/sbom/cli.py b/scripts/sbom/sbom/cli.py index 577161fae8..c73631e1f2 100644 --- a/scripts/sbom/sbom/cli.py +++ b/scripts/sbom/sbom/cli.py @@ -212,6 +212,10 @@ def main(argv: list[str] | None = None) -> int: licensing.ExpressionReadError, discovery.ArchiveError, discovery.SupplierError, + # Raised both for a build tree with no file API reply and for a repo + # root git cannot answer for; its message is the instruction to the + # operator, and a traceback buries it. + discovery.FileApiError, wheelfile.WheelError, validate_module.ManifestError, ) as error: diff --git a/scripts/sbom/sbom/discovery.py b/scripts/sbom/sbom/discovery.py index e08d3cd4a1..7751ee8003 100644 --- a/scripts/sbom/sbom/discovery.py +++ b/scripts/sbom/sbom/discovery.py @@ -32,7 +32,6 @@ from dataclasses import dataclass, replace from pathlib import Path -from elftools.common.exceptions import ELFError from packageurl import PackageURL from . import elf, licensing @@ -49,6 +48,10 @@ } ARCHIVE_SUFFIXES = (".tar.gz", ".tgz", ".tar.xz", ".tar.bz2", ".tar", ".zip") _MAX_ARCHIVE_BYTES = 1 << 31 +# How much of a library nested in an archive is kept, to read its build-id from. +# An SDK ships its libraries under `lib/`, not at the archive root, so without +# this the only members with a build-id are the ones nothing links against. +_ELF_HEAD_BYTES = 1 << 16 _GITHUB_REMOTE = re.compile( r"github\.com[:/](?P[^/]+)/(?P[^/]+?)(?:\.git)?/?$" @@ -959,14 +962,12 @@ def add_archive(self, path: Path, display: str) -> None: "member_count": len(members), } for member, data in members: - if elf.is_elf(data[:4]): - try: - build_id = elf.read_dynamic(data).build_id - except ELFError: # a third-party archive may hold a broken ELF - build_id = None - if build_id: - member = replace(member, build_id=build_id) - self.all_by_build_id.setdefault(build_id, []).append(member) + # `data` is the whole member only at the archive root; elsewhere it + # is the head kept for exactly this, so ask only for the build-id. + build_id = elf.build_id_of(data) + if build_id: + member = replace(member, build_id=build_id) + self.all_by_build_id.setdefault(build_id, []).append(member) self.by_hash.setdefault(member.sha256, member) self.all_by_hash.setdefault(member.sha256, []).append(member) self.by_name.setdefault(Path(member.path).name, []).append(member) @@ -997,22 +998,26 @@ def _read_members(path: Path, display: str) -> list[tuple[Member, bytes]]: collected: list[tuple[Member, bytes]] = [] def take(name: str, handle) -> None: - # Only a root member's bytes are kept -- for its licence text or its - # ELF header -- so the rest stream through the digest. Reading every - # member whole sizes peak memory by the largest file in the archive. + # A root member's bytes are kept whole, for its licence text; every + # other member streams through the digest, and keeps a head only if + # it is a library, for its build-id. Reading every member whole + # sizes peak memory by the largest file in the archive. at_root = Path(name).parent.as_posix() in {".", ""} digest = hashlib.sha256() size = 0 kept = bytearray() + head = b"" for chunk in iter(lambda: handle.read(1 << 20), b""): digest.update(chunk) - size += len(chunk) if at_root: kept += chunk + elif not size and elf.is_elf(chunk): + head = chunk[:_ELF_HEAD_BYTES] + size += len(chunk) collected.append( ( Member(display, name, digest.hexdigest(), size), - bytes(kept) if at_root else b"", + bytes(kept) if at_root else head, ) ) @@ -1069,7 +1074,10 @@ def discover_archives( index = ArchiveIndex() skipped = tuple(item.resolve() for item in skip) for path in _walk(repo_root): - if any(str(path).startswith(str(item)) for item in skipped): + # By path segment, not by string prefix: `dist` prefixes `distribution` + # and a build directory named `build` prefixes `buildtools`, and an + # archive skipped that way goes on to explain nothing. + if any(path.is_relative_to(item) for item in skipped): continue name = path.name.lower() if not name.endswith(ARCHIVE_SUFFIXES): diff --git a/scripts/sbom/sbom/document.py b/scripts/sbom/sbom/document.py index a8debc3d75..49947c0aa5 100644 --- a/scripts/sbom/sbom/document.py +++ b/scripts/sbom/sbom/document.py @@ -105,10 +105,9 @@ def _extracted_licenses( grants = licensing.obligation_texts(component.evidence) if not refs or not grants: continue + body = licensing.verbatim_terms(component.evidence) for ref in refs: - if ref in entries and entries[ref]["extractedText"] != "\n\n".join( - item.text for item in grants - ): + if ref in entries and entries[ref]["extractedText"] != body: raise ValueError( f"{ref} was minted for two components with different terms; " "a LicenseRef identifies one text" @@ -116,7 +115,7 @@ def _extracted_licenses( entries[ref] = { "licenseId": ref, "name": f"License terms shipped with {component.name}", - "extractedText": "\n\n".join(item.text for item in grants), + "extractedText": body, "comment": ( ( "Matched no text in the SPDX reference corpus" diff --git a/scripts/sbom/sbom/elf.py b/scripts/sbom/sbom/elf.py index 73d95f1bda..8769c9bbf9 100644 --- a/scripts/sbom/sbom/elf.py +++ b/scripts/sbom/sbom/elf.py @@ -15,6 +15,7 @@ from __future__ import annotations import io +import struct from dataclasses import dataclass from pathlib import Path @@ -55,6 +56,21 @@ def _build_id_from(elffile: ELFFile) -> str | None: return None +def build_id_of(data: bytes) -> str | None: + """The build-id of an in-memory image, which may be only its head. + + The note the linker writes sits with the program headers, so a bounded + prefix of a large library answers this without holding the whole file. A + prefix too short to reach the note reads as no build-id rather than raising. + """ + if not is_elf(data): + return None + try: + return _build_id_from(ELFFile(io.BytesIO(data))) + except (ELFError, OSError, struct.error): + return None + + def read_build_id(path: Path) -> str | None: """The build-id of a file on disk, read without loading it. diff --git a/scripts/sbom/sbom/evidence.py b/scripts/sbom/sbom/evidence.py index 39a7a7d981..21f8201a43 100644 --- a/scripts/sbom/sbom/evidence.py +++ b/scripts/sbom/sbom/evidence.py @@ -5,6 +5,7 @@ from __future__ import annotations +import hashlib import platform import re from dataclasses import dataclass @@ -391,6 +392,13 @@ def document( "license_concluded": component.license_concluded, "license_declared": component.license_declared, "evidence": [item.as_json() for item in component.evidence], + # The digest of the terms verbatim, as the document carries + # them. A component can state its licence across several files, + # and the join has no digest of its own for the verifier to + # compare against unless the build states one. + "verbatim_terms_sha256": hashlib.sha256( + licensing.verbatim_terms(component.evidence).encode("utf-8") + ).hexdigest(), "in_this_wheel": key in inventory.components_present, } for key, component in sorted(components.items()) diff --git a/scripts/sbom/sbom/licensing.py b/scripts/sbom/sbom/licensing.py index 591e66a6e2..cf05e566d6 100644 --- a/scripts/sbom/sbom/licensing.py +++ b/scripts/sbom/sbom/licensing.py @@ -116,6 +116,17 @@ def obligation_texts(evidence) -> list[LicenseEvidence]: return [item for item in evidence if satisfies_obligation(item.kind)] +def verbatim_terms(evidence) -> str: + """The body a LicenseRef carries for a component: every text it read. + + One component can state its terms across several files, so this is a join + and not a file. The build records its digest and the verifier compares + against that, which only works while both sides join the same way -- so they + both call this. + """ + return "\n\n".join(item.text for item in obligation_texts(evidence)) + + _DIGITS = re.compile(r"\d+") @@ -626,9 +637,15 @@ def canonical_evidence( seen: set[str] = set() entries = corpus()[1] for expression in sorted(expressions): - for token in _spdx_licensing().license_keys( - _spdx_licensing().parse(expression, validate=False) - ): + try: + parsed = _spdx_licensing().parse(expression, validate=False) + except (ExpressionError, ValueError, TypeError): + # A header tag is written by hand and can say anything -- "Public + # domain or MIT License" is not an expression. One that will not + # parse names no reference text; the component is then reported with + # no licence evidence, which is the gap, rather than stopping here. + continue + for token in _spdx_licensing().license_keys(parsed): reference = entries.get(token) if token in seen or reference is None or not reference.text: continue diff --git a/scripts/sbom/sbom/validate.py b/scripts/sbom/sbom/validate.py index ea06f14fdc..0d7b602d29 100644 --- a/scripts/sbom/sbom/validate.py +++ b/scripts/sbom/sbom/validate.py @@ -420,23 +420,30 @@ def _document_agrees_with_evidence( f"the build recorded {expected!r}" ) - # And the text that id stands for has to be one the build read. Where a - # component's terms came from several files the document joins them, so - # there is no single digest to compare and the join is left to the - # packaged-path check. - single = { - component["evidence"][0]["sha256"] - for component in components.values() - if len(component.get("evidence", [])) == 1 + # And the text a LicenseRef stands for has to be the terms the build read + # for some component. A component can state its licence across several + # files, so the body is a join with no digest of its own -- the evidence + # states that digest, which is what makes every entry checkable, shared ids + # included. An id minted for one component is held to that component's. + by_ref = {licensing.license_ref(key): key for key in components} + stated = { + key: component.get("verbatim_terms_sha256") + for key, component in components.items() } + any_stated = {digest for digest in stated.values() if digest} for extracted in spdx.get("hasExtractedLicensingInfos", []): + if not any_stated: + # Evidence written before this digest existed cannot answer. + break digest = hashlib.sha256( extracted.get("extractedText", "").encode("utf-8") ).hexdigest() - if single and digest not in recorded and digest not in single: + owner = by_ref.get(extracted["licenseId"]) + expected = {stated[owner]} if owner and stated.get(owner) else any_stated + if digest not in expected: failures.append( - f"{extracted['licenseId']}: the text it carries is not one the " - "build read" + f"{extracted['licenseId']}: the text it carries is not the terms " + "the build read" + (f" for {owner}" if owner else "") ) return failures @@ -547,7 +554,16 @@ def _check_manifest( published = beside / record["sidecars"]["spdx"]["filename"] if published.is_file(): with zipfile.ZipFile(wheel) as archive: - if archive.read(record["sbom_in_wheel"]) != published.read_bytes(): + embedded = record["sbom_in_wheel"] + # Reported, not raised: the member name comes out of the manifest, + # so one the wheel does not hold is a disagreement between the two + # and reads as such, rather than as a KeyError from the zip reader. + if embedded not in archive.namelist(): + failures.append( + f"the manifest names {embedded} as this wheel's SBOM, and the " + "wheel carries no such member" + ) + elif archive.read(embedded) != published.read_bytes(): failures.append( "the published SBOM copy differs from the one embedded in the wheel" ) diff --git a/scripts/sbom/sbom/wheelfile.py b/scripts/sbom/sbom/wheelfile.py index 18eb0ba321..bba2cf6f85 100644 --- a/scripts/sbom/sbom/wheelfile.py +++ b/scripts/sbom/sbom/wheelfile.py @@ -22,6 +22,7 @@ from email import message_from_bytes from email.generator import BytesGenerator from email.policy import compat32 +from elftools.common.exceptions import ELFError from installer.records import Hash, RecordEntry, parse_record_file from packaging.metadata import RawMetadata, parse_email from packaging.version import Version @@ -129,6 +130,17 @@ def _dist_info(names: list[str]) -> str: return candidates[0] +def dist_info_of(wheel_path: Path) -> str: + """The `.dist-info` directory name, from the archive's index alone. + + For a caller that wants only the metadata: `scan` hashes every member, which + over a release's worth of wheels is a full read of each for a name the + central directory already states. + """ + with zipfile.ZipFile(wheel_path) as archive: + return _dist_info(archive.namelist()) + + def scan(wheel_path: Path, analyze_elf: bool = False) -> WheelInfo: """Hash every member, and read the dynamic section of ELF members on request. @@ -166,7 +178,13 @@ def scan(wheel_path: Path, analyze_elf: bool = False) -> WheelInfo: data = handle.read() soname = needed = machine = build_id = None if analyze_elf and elf.is_elf(data[:4]): - dynamic = elf.read_dynamic(data) + try: + dynamic = elf.read_dynamic(data) + except ELFError as error: + raise WheelError( + f"{info.filename} begins with the ELF magic but cannot be " + f"read as one: {error}" + ) from error soname, needed, machine, build_id = ( dynamic.soname, dynamic.needed, diff --git a/tests/python/sbom/test_wheel_sbom.py b/tests/python/sbom/test_wheel_sbom.py index 92084c8a04..64392bd07f 100644 --- a/tests/python/sbom/test_wheel_sbom.py +++ b/tests/python/sbom/test_wheel_sbom.py @@ -372,3 +372,83 @@ def test_a_requirement_this_run_also_built_points_at_its_own_document( for item in spdx["relationships"] ) assert validate_module.check(companion) == [] + + +def _rewrite_document(wheel, mutate): + """Put a modified SBOM back into a wheel, under its own name.""" + info = wheelfile.scan(wheel) + name = wheelfile.sbom_member(info.dist_info, wheel.name) + with zipfile.ZipFile(wheel) as archive: + spdx = json.loads(archive.read(name)) + assert mutate(spdx), "nothing to mutate" + body = json.dumps(spdx, indent=2, sort_keys=True).encode() + b"\n" + staged = wheel.parent / f"staged-{wheel.name}" + wheelfile.rewrite( + wheel, + staged, + additions={}, + replacements={name: body}, + dist_info=info.dist_info, + ) + staged.replace(wheel) + + +def test_a_components_terms_may_span_several_files(workspace, license_data): + """A component can state its licence in more than one file. + + The LicenseRef then carries the join, which has no digest of its own, so the + build states one. Holding the entry to a single file's digest instead failed + a correct wheel. + """ + from sbom import licensing + + licensing.load_corpus(license_data) + root = workspace.build / "_deps" / "alpha-src" + root.mkdir(parents=True, exist_ok=True) + (root / "LICENSE").write_text(synth.PROPRIETARY_LICENSE) + (root / "COPYING").write_text(synth.PROPRIETARY_LICENSE + "\nAnd a second file.") + synth.track(workspace.root) + synth.write_wheel(workspace.wheel, synth.wheel_payload(workspace)) + + out = workspace.root / "sbom" + build_module.build(workspace.root, workspace.build, workspace.wheel, out) + + stem = WHEEL_NAME.removesuffix(".whl") + evidence_path = out / f"{stem}.build-evidence.json" + evidence = json.loads(evidence_path.read_text(encoding="utf-8")) + texts = [ + item + for item in evidence["components"]["alpha"]["evidence"] + if item["kind"] in ("grant", "pool") + ] + assert len(texts) > 1, "the component was set up to state its terms twice" + + spdx = _embedded(workspace.wheel) + joined = next( + item + for item in spdx["hasExtractedLicensingInfos"] + if "alpha" in item["licenseId"] + ) + assert joined["extractedText"].count(synth.PROPRIETARY_LICENSE) >= 1 + assert validate_module.check(workspace.wheel, build_evidence=evidence_path) == [] + + +def test_a_rewritten_licence_ref_body_is_caught_however_it_was_minted(built): + """The verbatim terms are what a LicenseRef stands for. + + A shared id belongs to no single component, which is not a reason to stop + checking it: that is the id the proprietary SDK terms travel under. + """ + swapped = "You may do whatever you like with this software." + + def rewrite(spdx): + for item in spdx["hasExtractedLicensingInfos"]: + item["extractedText"] = swapped + return bool(spdx["hasExtractedLicensingInfos"]) + + _rewrite_document(built["wheel"], rewrite) + + failures = validate_module.check( + built["wheel"], build_evidence=built["evidence_path"] + ) + assert any("not the terms the build read" in item for item in failures), failures From f5a508802a7fd274227b980129a913fb947fdea6 Mon Sep 17 00:00:00 2001 From: Andrei Aristarkhov Date: Thu, 1 Oct 2026 02:42:56 -0700 Subject: [PATCH 052/101] build(sbom): make a distribution ship the licence it declares The transition wheel declared `License-Expression: Apache-2.0` and carried no Apache-2.0 text. The gate covers what a build redistributes, and a distribution's own declared licence is not that, so nothing looked -- while the collector added `License-File: THIRD-PARTY-NOTICES.md` as the wheel's only such entry, so a PEP 639 reader resolved the declaration to a 232-byte stub. The notices file then pointed at a `third-party/` directory the wheel does not contain, because it writes that sentence before finding out there is nothing to put there. The declared expression is now an obligation like any other: its reference text is packaged from the pinned SPDX list where the wheel does not already ship one, and an identifier left with no text stops the build. The notices file says when a distribution redistributes nothing. Licence, homepage and supplier come from the wheel's own metadata rather than constants in the collector. A second distribution built from this tree is a different project, and the checkout's `pyproject.toml` describes only the one it is the root of -- the supplier comment said as much and the other two fields were hardcoded anyway. Both projects now declare their homepage, which is where that fact belongs. Signed-off-by: Andrei Aristarkhov --- pyproject.toml | 3 + scripts/sbom/sbom/build.py | 111 ++++++++++++++++++++++++++++++--- scripts/sbom/sbom/licensing.py | 9 +++ scripts/sbom/sbom/report.py | 21 +++++-- src/compat/pyproject.toml | 6 ++ tests/python/sbom/synth.py | 2 + 6 files changed, 138 insertions(+), 14 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index b220d0664f..0999ef7f35 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -28,6 +28,9 @@ dependencies = [ "pyyaml>=6.0.3", ] +[project.urls] +Homepage = "https://github.com/NVIDIA/IsaacCapture" + [project.optional-dependencies] dev = [ "pytest>=7.0.0", diff --git a/scripts/sbom/sbom/build.py b/scripts/sbom/sbom/build.py index 9fd1210be9..0f66928828 100644 --- a/scripts/sbom/sbom/build.py +++ b/scripts/sbom/sbom/build.py @@ -25,17 +25,42 @@ from packaging.utils import canonicalize_name from .discovery import project_supplier, sha256_file +from .document import NOASSERTION from .inventory import Attribution from .wheelfile import Entry -# Supplier is absent on purpose: it is read from the project's own packaging -# metadata at build time, so a fork or a rename cannot leave this file naming -# somebody else's organisation with nothing to catch it. -PROJECT = { - "homepage": "https://github.com/NVIDIA/IsaacCapture", - "license": "Apache-2.0", - "document_namespace": "https://github.com/NVIDIA/IsaacCapture/spdx", -} +# Only the namespace this tool mints documents under. Supplier, licence and +# homepage are read from the wheel's own packaging metadata, so a fork, a rename +# or a second distribution built from the same tree cannot leave this file +# stating facts about somebody else's. +PROJECT = {"document_namespace": "https://github.com/NVIDIA/IsaacCapture/spdx"} + + +def _project_facts(repo_root: Path, metadata: dict) -> dict: + """What this distribution says about itself, from its own metadata.""" + urls = metadata.get("project_urls") or {} + homepage = next( + ( + value + for key, value in urls.items() + if key.lower() in ("homepage", "home-page", "source", "repository") + ), + metadata.get("home_page") or NOASSERTION, + ) + # The wheel's own `Author` first: a second distribution built from this tree + # is a different project, and the checkout's `pyproject.toml` describes only + # the one it is the root of. + author = (metadata.get("author") or "").strip() + return { + **PROJECT, + "supplier": f"Organization: {author}" + if author + else project_supplier(repo_root), + # `license` is the legacy free-text field; only an expression is an + # expression, and a wheel declaring neither declares nothing. + "license": metadata.get("license_expression") or NOASSERTION, + "homepage": homepage, + } class BuildError(Exception): @@ -56,6 +81,37 @@ def default_license_data(build_dir: Path) -> Path: return build_dir / "_deps" / "license-list-data-src" / "json" +def _declared_without_text(declared: str, shipped: set[str]) -> list[str]: + """Identifiers in the declared expression with no packaged text to match.""" + if declared == NOASSERTION: + return [] + try: + identifiers = licensing.identifiers_in(declared) + except licensing.ExpressionReadError: + # An expression nothing can read names nothing that can be checked, and + # the document carries it verbatim; say so rather than pass silently. + return [declared] + return sorted(item for item in identifiers if f"{item.lower()}.txt" not in shipped) + + +def _own_license_texts(declared: str, shipped: set[str]) -> dict[str, str]: + """Reference texts for the ids this distribution declares and does not ship. + + The identifiers come from the wheel's own `License-Expression`, so this adds + the terms the distribution says apply to itself -- never a third party's, + which are packaged from the files those components actually carry. + """ + if declared == NOASSERTION: + return {} + names = {Path(item).name.lower() for item in shipped} + wanted = {} + for item in licensing.canonical_evidence("this-distribution", {declared}): + identifier = item.identified + if identifier and f"{identifier.lower()}.txt" not in names: + wanted[identifier] = item.text + return wanted + + def in_dependency_order(wheels: list[Path]) -> list[Path]: """Wheels of one run, each after any sibling it requires. @@ -141,7 +197,7 @@ def build( f"{wheel_path.name} already carries {sbom_name}. Run this on a freshly " "repaired wheel; rewriting one twice would duplicate its members." ) - metadata_raw, _ = wheelfile.read_metadata(wheel_path, wheel.dist_info) + metadata_raw, wheel_metadata = wheelfile.read_metadata(wheel_path, wheel.dist_info) resolver = inventory_module.Resolver(discovery, system_resolver) inventory = inventory_module.build( @@ -183,6 +239,26 @@ def build( report_module.declared_license_path(wheel.dist_info, target) ) + # The distribution's own declared licence is an obligation like any other. + # Apache-2.0 asks that recipients get a copy, and the gate above covers only + # what the build redistributes -- so a wheel could declare a licence whose + # text it does not carry, and the transition wheel did. + declared = _project_facts(repo_root, wheel_metadata)["license"] + shipped = { + report_module.declared_license_path(wheel.dist_info, item.name) + for item in wheel.entries + if report_module.is_packaged_license(wheel.dist_info, item.name) + } | set(packaged_paths) + missing_own = _own_license_texts(declared, shipped) + for identifier, text in sorted(missing_own.items()): + target = ( + f"{report_module.licenses_root(wheel.dist_info)}LICENSES/{identifier}.txt" + ) + additions[target] = text.encode("utf-8") + packaged_paths.append( + report_module.declared_license_path(wheel.dist_info, target) + ) + notices_path = report_module.notices_path(wheel.dist_info) additions[notices_path] = report_module.notices_markdown( inventory, wheel.dist_info, wheel_path.name @@ -191,6 +267,21 @@ def build( report_module.declared_license_path(wheel.dist_info, notices_path) ) + # Whatever could not be supplied from the corpus -- a LicenseRef, an id the + # pinned list does not carry -- is a declaration with no terms behind it. + unmet = _declared_without_text( + declared, + {Path(item).name.lower() for item in packaged_paths} + | {Path(item).name.lower() for item in shipped}, + ) + if unmet: + raise BuildError( + f"{wheel_path.name} declares {declared} and packages no text for " + + ", ".join(unmet) + + ".\nA distribution has to ship the terms it declares, the same as " + "anything it redistributes; obtain the text before publishing it." + ) + patched_metadata = wheelfile.add_license_files(metadata_raw, sorted(packaged_paths)) metadata_declared = patched_metadata != metadata_raw replacements = {f"{wheel.dist_info}/METADATA": patched_metadata} @@ -242,7 +333,7 @@ def build( evidence_doc["system_libraries"] = resolver.system_libraries spdx = document_module.build_document( - {**PROJECT, "supplier": project_supplier(repo_root)}, + _project_facts(repo_root, wheel_metadata), evidence_doc, projected_wheel, inventory, diff --git a/scripts/sbom/sbom/licensing.py b/scripts/sbom/sbom/licensing.py index cf05e566d6..180e45e0a8 100644 --- a/scripts/sbom/sbom/licensing.py +++ b/scripts/sbom/sbom/licensing.py @@ -666,6 +666,15 @@ class ExpressionReadError(Exception): """An expression in a document cannot be read, so what it names is unknown.""" +def identifiers_in(expression: str) -> list[str]: + """Every licence identifier an expression names, LicenseRef included.""" + try: + parsed = _spdx_licensing().parse(expression, validate=False) + except (ExpressionError, ValueError, TypeError) as error: + raise ExpressionReadError(f"{expression!r}: {error}") from error + return sorted(_spdx_licensing().license_keys(parsed)) + + def license_refs(*expressions: str) -> list[str]: """The LicenseRef- identifiers an expression uses, per the SPDX grammar.""" found: set[str] = set() diff --git a/scripts/sbom/sbom/report.py b/scripts/sbom/sbom/report.py index d0e259efa0..8e514bcf19 100644 --- a/scripts/sbom/sbom/report.py +++ b/scripts/sbom/sbom/report.py @@ -89,9 +89,18 @@ def _roles(inventory: Inventory, key: str) -> str: def notices_markdown(inventory: Inventory, dist_info: str, wheel_name: str) -> str: """The notice file that travels inside the wheel.""" - lines = [ - "# Third-party notices", - "", + lines = ["# Third-party notices", ""] + if not inventory.components_present: + # Saying where texts live when none were written sends a reader to a + # directory the wheel does not contain. + lines += [ + f"`{wheel_name}` redistributes no third-party component, so no " + "license text is packaged here.", + "", + ] + return "\n".join(lines) + + lines += [ f"Components redistributed in `{wheel_name}`, with the license text each", "obligation was read from. Texts are packaged under", f"`{licenses_root(dist_info)}third-party/`.", @@ -275,6 +284,10 @@ def report_markdown(payload: dict) -> str: ] lines.append("") if not gaps and not unidentified: - lines.append("None; every redistributed component ships an identified license.") + lines.append( + "None; every redistributed component ships an identified license." + if payload["components"] + else "This distribution redistributes no third-party component." + ) return "\n".join(lines) + "\n" diff --git a/src/compat/pyproject.toml b/src/compat/pyproject.toml index 1c373fac26..25f6433c6b 100644 --- a/src/compat/pyproject.toml +++ b/src/compat/pyproject.toml @@ -11,8 +11,14 @@ dynamic = ["version"] description = "Transition package for the rename to isaaccapture" requires-python = ">=3.11" license = "Apache-2.0" +authors = [ + { name = "NVIDIA" }, +] dependencies = ["isaaccapture>=1.6.dev0,<1.9"] +[project.urls] +Homepage = "https://github.com/NVIDIA/IsaacCapture" + [project.optional-dependencies] dev = ["isaaccapture[dev]>=1.6.dev0,<1.9"] ui = ["isaaccapture[ui]>=1.6.dev0,<1.9"] diff --git a/tests/python/sbom/synth.py b/tests/python/sbom/synth.py index 8935569e74..80c004a7ee 100644 --- a/tests/python/sbom/synth.py +++ b/tests/python/sbom/synth.py @@ -522,6 +522,8 @@ def _file_api_reply(root: Path, build: Path, extension: str) -> None: "Name: isaaccapture", "Version: 0.4.0", "Summary: Isaac Capture", + "Author: Example Org", + "Project-URL: Homepage, https://example.com/isaaccapture", "License-Expression: Apache-2.0", "License-File: LICENSE.md", "Requires-Python: >=3.11", From 55c27024f7999e3ca5e2f3bd440eb1a849b7a4ea Mon Sep 17 00:00:00 2001 From: Andrei Aristarkhov Date: Thu, 1 Oct 2026 02:50:32 -0700 Subject: [PATCH 053/101] build(sbom): name every copyright holder, and say what the wheel carries MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Four ways a notice went missing, each found against a real licence file in this build: only the first of several was read, so glfw lost Camilla Löwy and miniz lost one of its two holders; a holder running on past a trailing comma was cut mid-sentence, so libccd lost its institution; a notice with no year was not a notice, so mcap lost Foxglove; and a notice not starting its line was not found at all, so qhull lost its years. Under MIT, BSD and Zlib the notice is the obligation, and these travel in the notices file the wheel ships. The wheel package concluded the licence it declares while carrying components under terms the build could not name -- a proprietary SDK EULA among them. Declared stays what the distribution says of itself; concluded is now that AND the terms of what it redistributes under its own. Components with byte-identical terms share a LicenseRef, and the renaming only happened inside the document renderer -- so the notices file packaged in the wheel named an id the SBOM in the same wheel did not define. The sharing is settled once, before anything renders. A reference text substituted for a component that ships none was scored against the corpus it came from and reported as a match at containment 1.00, coverage 1.00. It is named by the identifier it was fetched for and says it is a substitution. A shortened file list says that it is shortened. Signed-off-by: Andrei Aristarkhov --- scripts/sbom/sbom/build.py | 17 +++++ scripts/sbom/sbom/discovery.py | 11 +-- scripts/sbom/sbom/document.py | 121 ++++++++++++++------------------- scripts/sbom/sbom/licensing.py | 118 ++++++++++++++++++++++++++++---- 4 files changed, 179 insertions(+), 88 deletions(-) diff --git a/scripts/sbom/sbom/build.py b/scripts/sbom/sbom/build.py index 0f66928828..a8165f7e0b 100644 --- a/scripts/sbom/sbom/build.py +++ b/scripts/sbom/sbom/build.py @@ -204,6 +204,23 @@ def build( wheel, metadata_raw, resolver, discovery.staged_root ) + # Components carrying byte-identical terms share one LicenseRef, settled here + # so the document, the reports and the notices packaged in the wheel all name + # the same id. Deciding it inside the renderer left the notices file naming + # an id the SBOM beside it did not define. + renamed = licensing.shared_refs(inventory.components_present.values()) + if renamed: + for key, component in inventory.components_present.items(): + inventory.components_present[key] = replace( + component, + license_concluded=licensing.rename_refs( + component.license_concluded, renamed + ), + license_declared=licensing.rename_refs( + component.license_declared, renamed + ), + ) + if inventory.unattributed: raise BuildError( "these wheel members could not be traced to anything this build " diff --git a/scripts/sbom/sbom/discovery.py b/scripts/sbom/sbom/discovery.py index 7751ee8003..bd2fd18ad6 100644 --- a/scripts/sbom/sbom/discovery.py +++ b/scripts/sbom/sbom/discovery.py @@ -176,8 +176,8 @@ def _notice_from(evidence) -> str: found for item in evidence if item.kind == "grant" - and (found := licensing.read_notice(item.text)) - and not licensing.notice_is_the_licence_authors( + for found in licensing.read_notices(item.text) + if not licensing.notice_is_the_licence_authors( found, [license_id for license_id, _, _ in item.matches] ) ] @@ -648,8 +648,11 @@ def discover_vendored( concluded, declared = licensing.expression(evidence, key) if licences.get(key): declared = licensing.combine(sorted(licences[key])) - relative = ", ".join( - sorted(str(item.relative_to(repo_root)) for item in paths)[:4] + # Say when the list is shortened; without the marker it reads as the + # whole set, and the first four by name need not be the ones that ship. + named = sorted(str(item.relative_to(repo_root)) for item in paths) + relative = ", ".join(named[:4]) + ( + f", and {len(named) - 4} more" if len(named) > 4 else "" ) components[key] = Component( key=key, diff --git a/scripts/sbom/sbom/document.py b/scripts/sbom/sbom/document.py index 49947c0aa5..72b4e36acf 100644 --- a/scripts/sbom/sbom/document.py +++ b/scripts/sbom/sbom/document.py @@ -55,6 +55,12 @@ def _attribution_texts(component: Component, dist_info: str) -> list[str]: for license_id, containment, coverage in item.matches ) if item.matches + # No text of its own to package: this is the SPDX reference text for + # the identifier the component's files state about themselves, which + # is a substitution and not a reading of the component. + else f" the SPDX reference text for {item.identified}, substituted " + "because this component ships no license file" + if item.origin == licensing.SPDX_REFERENCE else "" ) for item in component.evidence @@ -93,10 +99,8 @@ def _component_package(component: Component, roles: set[str], dist_info: str) -> return package -def _extracted_licenses( - components: list[Component], -) -> tuple[list[dict], dict[str, str]]: - """Texts for every LicenseRef the document uses, taken from the shipped file.""" +def _extracted_entries(components: list[Component]) -> list[dict]: + """One entry per distinct text, named by the id the components now share.""" entries: dict[str, dict] = {} for component in components: refs = licensing.license_refs( @@ -112,9 +116,18 @@ def _extracted_licenses( f"{ref} was minted for two components with different terms; " "a LicenseRef identifies one text" ) + shipped = ", ".join(item.path for item in grants) entries[ref] = { "licenseId": ref, - "name": f"License terms shipped with {component.name}", + "name": "License terms shipped with " + + ", ".join( + sorted( + {entries[ref]["_for"], component.name} + if ref in entries + else {component.name} + ) + ), + "_for": component.name, "extractedText": body, "comment": ( ( @@ -125,64 +138,12 @@ def _extracted_licenses( else "Matched an identifier newer than the SPDX grammar " "this document ships with" ) - + "; reproduced here verbatim from " - + ", ".join(item.path for item in grants) - + "." + + f"; reproduced here verbatim from {shipped}." ), } - return _share_identical_texts([entries[key] for key in sorted(entries)]) - - -def _rename_refs(expression: str, renamed: dict[str, str]) -> str: - """Substitute whole identifiers, never substrings. - - `LicenseRef-alpha` is a prefix of `LicenseRef-alpha-extra`, so a plain - replace turns the longer id into one nothing defines and orphans its entry. - """ - for old_id, new_id in renamed.items(): - expression = re.sub( - rf"(? tuple[list[dict], dict[str, str]]: - """One entry per distinct text, not one per component that ships it. - - Components can carry byte-identical terms -- two builds of one SDK, a licence - a vendor applies across products -- and a LicenseRef is an identifier for a - text, so minting one each duplicates the text in full. Where several share - one, they share one id, named after the text so it belongs to no single - component. - """ - by_text: dict[str, list[dict]] = {} - for entry in entries: - by_text.setdefault(entry["extractedText"], []).append(entry) - - shared: list[dict] = [] - renamed: dict[str, str] = {} - for text, group in by_text.items(): - if len(group) == 1: - shared.append(group[0]) - continue - digest = hashlib.sha256(text.encode("utf-8")).hexdigest()[:12] - canonical = f"LicenseRef-shared-text-{digest}" - for entry in group: - renamed[entry["licenseId"]] = canonical - shared.append( - { - **group[0], - "licenseId": canonical, - "name": "License terms shipped with " - + ", ".join( - sorted( - item["name"].removeprefix("License terms shipped with ") - for item in group - ) - ), - } - ) - return sorted(shared, key=lambda item: item["licenseId"]), renamed + for entry in entries.values(): + entry.pop("_for", None) + return sorted(entries.values(), key=lambda item: item["licenseId"]) _EXTRA_MARKER = re.compile(r"\bextra\s*==") @@ -247,6 +208,28 @@ def _dependency_refs( return refs +def _wheel_conclusion(declared: str, inventory: Inventory) -> str: + """The declared licence, AND the terms of everything redistributed under its own. + + A component whose terms the build could not name travels as a LicenseRef, + and those are exactly the ones a reader must not miss -- a proprietary SDK + EULA among them. Components under a licence the declaration already covers + add nothing and are left out. + """ + others = sorted( + { + component.license_concluded + for component in inventory.components_present.values() + if "LicenseRef-" in component.license_concluded + } + ) + if not others: + return declared + if declared == NOASSERTION: + return licensing.combine(others) + return licensing.combine([declared, *others]) + + def build_document( project: dict, evidence_doc: dict, @@ -335,7 +318,11 @@ def build_document( "downloadLocation": NOASSERTION, "filesAnalyzed": True, "homepage": project["homepage"], - "licenseConcluded": project["license"], + # Declared is what the distribution says of itself. Concluded is what + # this document found in it, and a wheel carrying a component under + # other terms is not wholly under the declared licence -- saying so is + # the field a compliance tool reads for "what is this artifact". + "licenseConcluded": _wheel_conclusion(project["license"], inventory), "licenseDeclared": project["license"], "licenseInfoFromFiles": sorted(stated_in_files) or [NOASSERTION], # The notices its own files carry. Declaring none beside a supplier, a @@ -525,15 +512,7 @@ def build_document( # rebuild be compared against a published one byte for byte. stamp = stamped_now() - extracted, renamed = _extracted_licenses( - list(inventory.components_present.values()) - ) - if renamed: - # Every reference to a text that is now shared points at the shared id; - # leaving one behind would use a LicenseRef the document stops defining. - for package in packages: - for key in ("licenseConcluded", "licenseDeclared"): - package[key] = _rename_refs(package[key], renamed) + extracted = _extracted_entries(list(inventory.components_present.values())) return { "spdxVersion": SPDX_VERSION, diff --git a/scripts/sbom/sbom/licensing.py b/scripts/sbom/sbom/licensing.py index 180e45e0a8..1bca73dd06 100644 --- a/scripts/sbom/sbom/licensing.py +++ b/scripts/sbom/sbom/licensing.py @@ -337,13 +337,22 @@ def _read(path: Path) -> str | None: return data.decode("utf-8", "replace") +SPDX_REFERENCE = "spdx-reference" + + def evidence_for( component: str, origin: str, display: str, text: str, kind: str ) -> LicenseEvidence: - matchable = kind == "grant" + # A reference text is not matched: scoring it against the corpus it came + # from returns a perfect score, which reads as a measurement of the + # component's own file and is a measurement of nothing. + matchable = kind == "grant" and origin != SPDX_REFERENCE tag = _SPDX_TAG.search(text[:4096]) if matchable else None matches = tuple(identify(text)) if matchable else () identified = " AND ".join(license_id for license_id, _, _ in matches) or None + if origin == SPDX_REFERENCE: + # Named by the identifier it was fetched for, with no score behind it. + identified = Path(display).stem return LicenseEvidence( component=component, origin=origin, @@ -468,6 +477,13 @@ def expression( grants = [item for item in evidence if item.kind == "grant"] identified = sorted( {license_id for item in grants for license_id, _, _ in item.matches} + # A reference text is named by the identifier it was fetched for; there + # is no match to read, because matching it would be circular. + | { + item.identified + for item in grants + if item.origin == SPDX_REFERENCE and item.identified + } ) if identified and component_key and not all(map(expressible, identified)): identified = [license_ref(component_key)] @@ -551,11 +567,19 @@ def fold_notices(notices) -> list[str]: ) +# A notice is `Copyright` plus a marker that prose does not carry: a (c), a ©, +# or a year. It need not start the line -- qhull writes `Qhull, Copyright (c) +# 1993-2020` -- so a short lead-in is allowed before it. _PLAIN_COPYRIGHT = re.compile( - r"^[^\w\n]*(?PCopyright\b[^\n\r]*\d{4}[^\n\r]*|" - r"Copyright\b[^\n\r]*)$", + r"^[^\n\r]{0,40}?(?PCopyright\b\s*(?:\(c\)|©|\d{4})[^\n\r]*)$", re.MULTILINE | re.IGNORECASE, ) +# Lines of a licence body that mention copyright without stating one. +_NOT_A_NOTICE = re.compile( + r"\bshall be\b|\bowner or entity\b|\bbe liable\b|\bmeans?\b|" + r"\bsubject to\b|\bdefined as\b", + re.IGNORECASE, +) _PLACEHOLDER = re.compile( r"\[yyyy\]||\[name of copyright owner\]| bool: ) -def read_notice(text: str) -> str | None: - """A concrete copyright line stated in a licence text, if it carries one. +def read_notices(text: str) -> list[str]: + """Every concrete copyright notice a licence text states. - A reference text carries the template instead -- `Copyright [yyyy] [name of - copyright owner]` -- which names nobody, so it is not a notice and recording - it would claim one where the file states none. + All of them: glfw names two holders and miniz names two, and under MIT, BSD + and Zlib the notice is the obligation, so keeping the first is a notice file + that does not discharge it. A reference text carries the template instead -- + `Copyright [yyyy] [name of copyright owner]` -- which names nobody. """ - for match in _PLAIN_COPYRIGHT.finditer(text): + lines = text.splitlines() + found: list[str] = [] + for index, line in enumerate(lines): + match = _PLAIN_COPYRIGHT.match(line) + if not match: + continue notice = " ".join(match.group("notice").split()).rstrip("*/ ") - if _PLACEHOLDER.search(notice) or not re.search(r"\d{4}", notice): + # A holder can run on across lines -- libccd names a department, a + # faculty and a university -- and a trailing comma is the author saying + # so. Anything else ends the notice. + step = index + while notice.endswith(",") and step + 1 < len(lines): + step += 1 + notice = f"{notice} {' '.join(lines[step].split())}".strip().rstrip("*/ ") + if _PLACEHOLDER.search(notice) or _NOT_A_NOTICE.search(notice): continue - return notice - return None + # Something has to follow the word: a holder, or at least the years the + # file states. qhull writes `Qhull, Copyright (c) 1993-2020` and puts the + # holder on the lines below, so a year alone is still what it states. + rest = notice[len("Copyright") :] + if not re.search(r"[A-Za-z]{2}", rest) and not re.search(r"\d{4}", rest): + continue + if notice not in found: + found.append(notice) + return found + + +def read_notice(text: str) -> str | None: + """The first notice a licence text states, where only one is wanted.""" + notices = read_notices(text) + return notices[0] if notices else None def read_copyright(text: str) -> str | None: @@ -653,7 +703,7 @@ def canonical_evidence( found.append( evidence_for( component_key, - "spdx-reference", + SPDX_REFERENCE, f"{token}.txt", reference.text, "grant", @@ -666,6 +716,48 @@ class ExpressionReadError(Exception): """An expression in a document cannot be read, so what it names is unknown.""" +def shared_refs(components) -> dict[str, str]: + """Rename map putting components with byte-identical terms on one LicenseRef. + + A LicenseRef identifies a text, not a component, so two builds of one SDK + under the same EULA share an id. The map is produced here, before anything + renders, so the document, the report and the packaged notices all name the + same id -- they are cross-referenced, and a text shipped under one name in + the wheel's notices and another in the wheel's SBOM reconciles with nothing. + """ + by_text: dict[str, list[str]] = {} + for component in components: + terms = verbatim_terms(component.evidence) + if terms: + by_text.setdefault(terms, []).append(component.key) + + renamed: dict[str, str] = {} + for text, keys in by_text.items(): + if len(keys) < 2: + continue + canonical = f"LicenseRef-shared-text-{sha256_text(text)[:12]}" + for key in keys: + renamed[license_ref(key)] = canonical + return renamed + + +def sha256_text(text: str) -> str: + return hashlib.sha256(text.encode("utf-8")).hexdigest() + + +def rename_refs(expression: str, renamed: dict[str, str]) -> str: + """Substitute whole identifiers, never substrings. + + `LicenseRef-alpha` is a prefix of `LicenseRef-alpha-extra`, so a plain + replace turns the longer id into one nothing defines and orphans its entry. + """ + for old_id, new_id in renamed.items(): + expression = re.sub( + rf"(? list[str]: """Every licence identifier an expression names, LicenseRef included.""" try: From 90b8ae3fa1c54d817a426ba7067fc698d5187426 Mon Sep 17 00:00:00 2001 From: Andrei Aristarkhov Date: Thu, 1 Oct 2026 02:57:58 -0700 Subject: [PATCH 054/101] build(sbom): credit a holder stated inside a component, not just at its root A component can carry terms for one subdirectory of itself -- a vendored asset set inside a larger project -- and only its root and `LICENSES/` were read, so the assets shipped credited to the component's own holder and not to theirs. Those files are collected now, kept only where that directory reached this wheel, and their holders travel in the notices. A test suite's licence is not this wheel's business; a shipped asset set's is. The settlements that follow -- which nested terms survive, and which components share a LicenseRef -- are applied to one set of components rather than to the inventory alone, because the evidence sidecar is written from the other view and the two disagreed about what a component carries. Three smaller claims the document could not support: a hostname is not a supplier, so an archive served from a CDN endpoint names the registrable domain or nothing; a licence section quoted out of a README is named with the fragment it is, not with the whole file whose digest it does not have; and a member matching a file in a staged tree says so, rather than calling itself derived from an input nothing recorded. Signed-off-by: Andrei Aristarkhov --- scripts/sbom/sbom/build.py | 79 +++++++++++++++++++++++------ scripts/sbom/sbom/discovery.py | 8 +-- scripts/sbom/sbom/evidence.py | 22 +++++--- scripts/sbom/sbom/inventory.py | 19 ++++++- scripts/sbom/sbom/licensing.py | 24 +++++++-- tests/python/sbom/test_discovery.py | 4 +- 6 files changed, 125 insertions(+), 31 deletions(-) diff --git a/scripts/sbom/sbom/build.py b/scripts/sbom/sbom/build.py index a8165f7e0b..49f92eaf3d 100644 --- a/scripts/sbom/sbom/build.py +++ b/scripts/sbom/sbom/build.py @@ -24,6 +24,7 @@ from packaging.requirements import Requirement from packaging.utils import canonicalize_name +from . import discovery as discovery_module from .discovery import project_supplier, sha256_file from .document import NOASSERTION from .inventory import Attribution @@ -81,6 +82,46 @@ def default_license_data(build_dir: Path) -> Path: return build_dir / "_deps" / "license-list-data-src" / "json" +def _with_reached_nested(inventory, components: dict) -> dict: + """Drop nested terms whose directory contributed nothing to this wheel. + + A nested LICENSE states a holder for part of a component -- the assets under + it -- and dropping it names the wrong holder for those bytes. Keeping all of + them names holders for code that never shipped. + """ + reached: dict[str, set[str]] = {} + for attribution in inventory.attributions.values(): + if attribution.source_path: + for key in attribution.components: + reached.setdefault(key, set()).add(attribution.source_path) + + updated = {} + for key, component in components.items(): + nested = [item for item in component.evidence if item.kind == "nested"] + if not nested: + continue + sources = reached.get(key, set()) + kept = [ + item + for item in nested + if any( + source.startswith(item.path.rsplit("/", 1)[0] + "/") + for source in sources + ) + ] + if len(kept) == len(nested): + continue + evidence = tuple( + item for item in component.evidence if item.kind != "nested" or item in kept + ) + updated[key] = replace( + component, + evidence=evidence, + copyright_text=discovery_module.notice_from(evidence), + ) + return updated + + def _declared_without_text(declared: str, shipped: set[str]) -> list[str]: """Identifiers in the declared expression with no packaged text to match.""" if declared == NOASSERTION: @@ -204,22 +245,28 @@ def build( wheel, metadata_raw, resolver, discovery.staged_root ) - # Components carrying byte-identical terms share one LicenseRef, settled here - # so the document, the reports and the notices packaged in the wheel all name - # the same id. Deciding it inside the renderer left the notices file naming - # an id the SBOM beside it did not define. - renamed = licensing.shared_refs(inventory.components_present.values()) - if renamed: - for key, component in inventory.components_present.items(): - inventory.components_present[key] = replace( - component, - license_concluded=licensing.rename_refs( - component.license_concluded, renamed - ), - license_declared=licensing.rename_refs( - component.license_declared, renamed - ), - ) + # Two settlements before anything renders, applied to both views of a + # component so the document, the reports, the packaged notices and the + # evidence sidecar all describe the same thing. + # + # Terms stated for one subdirectory apply only if that directory reached + # this wheel: a test suite's licence is not this wheel's business, a + # vendored asset set's is. And components carrying byte-identical terms + # share one LicenseRef -- settled here because deciding it in the renderer + # left the notices file naming an id the SBOM beside it did not define. + settled = _with_reached_nested(inventory, inventory.components_present) + present = {**inventory.components_present, **settled} + renamed = licensing.shared_refs(present.values()) + for key, component in list(present.items()): + present[key] = replace( + component, + license_concluded=licensing.rename_refs( + component.license_concluded, renamed + ), + license_declared=licensing.rename_refs(component.license_declared, renamed), + ) + inventory.components_present.update(present) + resolver.components.update(present) if inventory.unattributed: raise BuildError( diff --git a/scripts/sbom/sbom/discovery.py b/scripts/sbom/sbom/discovery.py index bd2fd18ad6..dd3014f2ea 100644 --- a/scripts/sbom/sbom/discovery.py +++ b/scripts/sbom/sbom/discovery.py @@ -161,7 +161,7 @@ def git(repo: Path, *args: str) -> str | None: ) -def _notice_from(evidence) -> str: +def notice_from(evidence) -> str: """The copyright a component's own licence text states, where it states one. Permissive licences generally require the notice to travel with the terms, @@ -175,7 +175,7 @@ def _notice_from(evidence) -> str: notices = [ found for item in evidence - if item.kind == "grant" + if item.kind in ("grant", "nested") for found in licensing.read_notices(item.text) if not licensing.notice_is_the_licence_authors( found, [license_id for license_id, _, _ in item.matches] @@ -267,7 +267,7 @@ def discover_source_trees(deps_dir: Path) -> dict[str, Component]: download_location=download, source_info=" ".join(details), root=root, - copyright_text=_notice_from(evidence), + copyright_text=notice_from(evidence), license_concluded=concluded, license_declared=declared_expression, evidence=tuple(evidence), @@ -903,7 +903,7 @@ def system_component(soname: str, origin: dict) -> Component: # The distro copyright file is where the holder is stated, and it was # read, packaged and put on the file record -- then left off the package # and out of the notices every other component appears in. - copyright_text=_notice_from(evidence), + copyright_text=notice_from(evidence), homepage="NOASSERTION", # A distro package the machine can name resolves; pkg:generic does not. purl=( diff --git a/scripts/sbom/sbom/evidence.py b/scripts/sbom/sbom/evidence.py index 21f8201a43..4785cc0c22 100644 --- a/scripts/sbom/sbom/evidence.py +++ b/scripts/sbom/sbom/evidence.py @@ -28,7 +28,7 @@ discover_vendored, project_authors, discover_source_trees, - _notice_from, + notice_from, tracked_files, EMPTY_SHA256, ) @@ -103,11 +103,21 @@ def _fetched_from(archive: Path) -> str | None: def _supplier_of(url: str | None) -> str: - """Whoever served the archive, named by the host that did.""" - if not url: + """Who supplied the archive, where the URL says so. + + A hostname is not a supplier: `api.ngc.nvidia.com` is a CDN endpoint, and + SPDX `supplier` is who a recipient asks for source or terms. Only a host + whose registrable domain names the supplier answers that, and the download + location carries the URL for everything else. + """ + host = urlparse(url).hostname if url else None + if not host: return "NOASSERTION" - host = urlparse(url).hostname - return f"Organization: {host}" if host else "NOASSERTION" + labels = host.split(".") + # Second-level label of a two-label public suffix (`nvidia` of + # `api.ngc.nvidia.com`); anything shorter is not a name. + name = labels[-2] if len(labels) >= 2 else "" + return f"Organization: {name}" if len(name) > 2 else "NOASSERTION" def _archive_components( @@ -153,7 +163,7 @@ def _archive_components( ), license_concluded=concluded, license_declared=declared, - copyright_text=_notice_from(evidence), + copyright_text=notice_from(evidence), evidence=tuple(evidence), ) return components diff --git a/scripts/sbom/sbom/inventory.py b/scripts/sbom/sbom/inventory.py index 67ee37c1ae..a97a180551 100644 --- a/scripts/sbom/sbom/inventory.py +++ b/scripts/sbom/sbom/inventory.py @@ -65,6 +65,10 @@ class Attribution: # the other made the same fact answerable two ways depending on which index # happened to hold the bytes. exact: bool = False + # Where in the component these bytes came from, where an index named it. + # The detail string says it in prose; this is the same fact in a form the + # rest of the collector can use to ask what else sits in that directory. + source_path: str | None = None def as_json(self) -> dict: return { @@ -72,6 +76,7 @@ def as_json(self) -> dict: "origin": self.origin, "detail": self.detail, "primary": self.primary, + "source_path": self.source_path, "components": { key: sorted(how) for key, how in sorted(self.components.items()) }, @@ -262,6 +267,7 @@ def _trace( detail=f"{source}; byte-identical to {component_file}", components={key: {"copied-file"}}, primary=key, + source_path=component_file, ) member = self.archives.by_hash.get(candidate_digest) @@ -290,13 +296,23 @@ def _trace( if by_path: build_file = by_path owner = self._staged_component_of(build_file) + # Say which of the two this is. The bytes match the build tree, so + # the member is that file; what the build made it from is not + # recorded anywhere this can read, and claiming a derivation without + # naming its input is the claim the document cannot support. + detail = f"{source}; byte-identical to {build_file}" + if owner: + detail += ( + f", in a tree the build staged for {owner}; the input it was" + " produced from is not recorded" + ) return Attribution( path=wheel_path, # Transforming a component's own file leaves a derivative work, # which carries that component's terms. Only output that owes # nothing to a third party is merely "generated". origin="derived" if owner else "generated", - detail=f"{source}; {build_file}", + detail=detail, components={owner: {"derived-file"}} if owner else {}, primary=owner, ) @@ -556,6 +572,7 @@ def _resolve_by_path( ), components={key: {"copied-file" if same else "derived-file"}}, primary=key, + source_path=component_file, ) build_file = self.build_files.path_suffix_match(wheel_path) diff --git a/scripts/sbom/sbom/licensing.py b/scripts/sbom/sbom/licensing.py index 1bca73dd06..1c181f238a 100644 --- a/scripts/sbom/sbom/licensing.py +++ b/scripts/sbom/sbom/licensing.py @@ -72,8 +72,9 @@ class LicenseEvidence: sha256: str size: int text: str - # "grant" is the component's own license, "notice" an attribution file, and - # "pool" a REUSE LICENSES/ entry that applies to individual files only. + # "grant" is the component's own license, "notice" an attribution file, + # "pool" a REUSE LICENSES/ entry that applies to individual files only, and + # "nested" terms stated for one subdirectory of a larger component. kind: str identified: str | None = None matches: tuple[tuple[str, float, float], ...] = () @@ -392,12 +393,26 @@ def discover_in_tree( (item, "pool") for item in sorted(entry.rglob("*")) if item.is_file() ) + # A subdirectory can carry its own terms and its own holder -- a vendored + # asset set inside a larger project. Collected as "nested" so it neither + # discharges the component's packaging obligation nor changes its licence, + # and kept only where that directory reached the wheel; see `nested_for`. + for path in sorted(root.rglob("*")) if root.is_dir() else []: + if ( + path.is_file() + and path.parent != root + and path.parent.name.upper() not in {"LICENSES", "LICENSE"} + and (classify(path.name) or _NOTICE_NAMES.match(path.name)) + ): + candidates.append((path, "nested")) + for path, kind in candidates: text = _read(path) if text is None: continue display = f"{display_prefix}/{path.relative_to(root).as_posix()}" found.append(evidence_for(component, "component-file", display, text, kind)) + # "nested" never satisfies the grant test below; that is the point. if not any(item.kind == "grant" for item in found): found.extend(_readme_fallback(component, root, display_prefix)) @@ -421,7 +436,10 @@ def _readme_fallback( # build instructions as the terms that shipped, and would bury a real # licence quoted there under enough prose to fall below the coverage # threshold and degrade to a LicenseRef. - display = f"{display_prefix}/{entry.name}" + # The fragment is load-bearing: the digest below is of the section, not + # of the file, and a path naming the whole file beside it is a pair a + # verifier cannot reproduce. + display = f"{display_prefix}/{entry.name}#license-section" return [ evidence_for( component, "readme-section", display, section.group(0), "grant" diff --git a/tests/python/sbom/test_discovery.py b/tests/python/sbom/test_discovery.py index 3eae431ee8..40e67a634e 100644 --- a/tests/python/sbom/test_discovery.py +++ b/tests/python/sbom/test_discovery.py @@ -249,7 +249,9 @@ def test_a_fetched_archive_records_where_it_came_from(workspace, license_data): after = known.components[synth.SDK_ARCHIVE] assert after.download_location.startswith("https://api.ngc.nvidia.com/") - assert after.supplier == "Organization: api.ngc.nvidia.com" + # The registrable name, not the endpoint: SPDX supplier is who a recipient + # asks for source or terms, and a CDN hostname answers nobody. + assert after.supplier == "Organization: nvidia" assert "fetched from https://" in after.source_info From bc781d3d90290243e6e859b267ffc3e50d936155 Mon Sep 17 00:00:00 2001 From: Andrei Aristarkhov Date: Thu, 1 Oct 2026 02:59:13 -0700 Subject: [PATCH 055/101] docs(sbom): record the rules this round added The design doc now covers what the collector does: terms stated for part of a component, the distribution's own declared licence as an obligation, and the difference between what a wheel declares and what this document concludes it carries. `docs/sbom-open-findings.md` is deliberately untracked; the ignore line keeps a `git add -A` from committing it. Signed-off-by: Andrei Aristarkhov --- .gitignore | 3 +++ docs/source/references/sbom.rst | 16 ++++++++++++++++ 2 files changed, 19 insertions(+) diff --git a/.gitignore b/.gitignore index c7ce6a078e..0bc18af015 100644 --- a/.gitignore +++ b/.gitignore @@ -90,3 +90,6 @@ tmp/ !/examples/mujoco_xr/python/isaacteleop_examples/mujoco_xr/assets/follower/follower_arm.xml # Wheel SBOM and license evidence written by the collector in scripts/sbom/ /sbom/ + +# Audit notes kept with the branch, deliberately not published. +docs/sbom-open-findings.md diff --git a/docs/source/references/sbom.rst b/docs/source/references/sbom.rst index 31d279bc8f..4371bccb55 100644 --- a/docs/source/references/sbom.rst +++ b/docs/source/references/sbom.rst @@ -206,6 +206,22 @@ decides. Where a component has *no* license file, and only states an identifier source headers, there is nothing of its own to package: the text that identifier names in the pinned list ships instead, together with the copyright line those headers state. +A component can also state terms for one part of itself — a vendored asset set inside a +larger project, with its own ``LICENSE`` and its own holder. Those are read and packaged +too, but only where that directory reached the wheel, and they never change the component's +own licence: they name a holder for the bytes that shipped, which is what the notice +obligation is about. + +The distribution's own declared licence is an obligation like any other. A wheel declaring +``License-Expression`` and shipping no text for it stops the build, the same as a +redistributed component with no terms; where the wheel ships none, the reference text for +each identifier is packaged from the pinned list and declared under PEP 639. + +``licenseDeclared`` on the wheel is what the distribution says of itself. ``licenseConcluded`` +is what this document found in it, so a wheel carrying a component whose terms the build +could not name states that expression alongside the declaration — a reader asking "what is +this artifact" is told about the proprietary payload rather than about the declaration only. + Never a file from this checkout. ``LICENSES/`` holds this repository's REUSE pool, whose copies exist to license *this* project and carry its notices; packaging one as a third party's terms would put our copyright above their code. Those texts ship under From 221575fffe3e4ff56e0d5e0819cdc6d8bf4f6157 Mon Sep 17 00:00:00 2001 From: Andrei Aristarkhov Date: Thu, 1 Oct 2026 03:11:30 -0700 Subject: [PATCH 056/101] build(sbom): say what the build knows about a sibling it cannot cite Two wheels of one run can require each other, and a document citing another's digest has to be written second -- so one of the pair got a digest-bound reference and the other reported its sibling as an install-time unknown, with version, supplier and licence unasserted for a distribution the same run had just built and described. Which one lost was a function of discovery order. The facts a wheel states about itself need no document, so they are read for every wheel of the run before any is described. A sibling already described is still referenced by document; one that cannot be says so, and states the rest. `check-set` is the only place both documents are in hand, so it is where a cross-document reference is now resolved rather than merely declared: the namespace must name a document in the release, the SHA1 must be that document's, and the element pointed at must exist in it. Also: `sourceInfo` no longer renders a missing homepage as a noun, and leaves out the architecture and build type for a wheel with nothing compiled in it; both the commit lines say when the tree had uncommitted changes, which is the difference between a build a reader can reproduce and one they cannot; the human-readable report lists the licence texts packaged in the wheel, which only its JSON twin carried; and a count is a number. Signed-off-by: Andrei Aristarkhov --- scripts/sbom/sbom/build.py | 23 +++++++ scripts/sbom/sbom/cli.py | 11 ++-- scripts/sbom/sbom/document.py | 98 +++++++++++++++++++++------- scripts/sbom/sbom/licensing.py | 6 +- scripts/sbom/sbom/report.py | 24 ++++++- scripts/sbom/sbom/validate.py | 68 +++++++++++++++++++ tests/python/sbom/synth.py | 3 + tests/python/sbom/test_wheel_sbom.py | 16 +++++ 8 files changed, 217 insertions(+), 32 deletions(-) diff --git a/scripts/sbom/sbom/build.py b/scripts/sbom/sbom/build.py index 49f92eaf3d..e02edb9f60 100644 --- a/scripts/sbom/sbom/build.py +++ b/scripts/sbom/sbom/build.py @@ -153,6 +153,29 @@ def _own_license_texts(declared: str, shipped: set[str]) -> dict[str, str]: return wanted +def siblings_of(wheels: list[Path], repo_root: Path) -> dict[str, dict]: + """What each wheel of this run states about itself, by canonical name. + + Read before any of them is described, so a requirement naming one is never + reported as an install-time unknown. Two wheels can require each other -- a + renamed project and its transition wheel do -- and only one of them can + carry a digest-bound reference to the other's document; both can state what + the metadata says, because that needs no document. + """ + known: dict[str, dict] = {} + for wheel in wheels: + _, parsed = wheelfile.read_metadata(wheel, wheelfile.dist_info_of(wheel)) + facts = _project_facts(repo_root, parsed) + known[canonicalize_name(parsed["name"])] = { + "name": parsed["name"], + "version": parsed.get("version") or NOASSERTION, + "supplier": facts["supplier"], + "license_declared": facts["license"], + "wheel": wheel.name, + } + return known + + def in_dependency_order(wheels: list[Path]) -> list[Path]: """Wheels of one run, each after any sibling it requires. diff --git a/scripts/sbom/sbom/cli.py b/scripts/sbom/sbom/cli.py index c73631e1f2..9c4cd8de68 100644 --- a/scripts/sbom/sbom/cli.py +++ b/scripts/sbom/sbom/cli.py @@ -70,10 +70,12 @@ def _cmd_build(args: argparse.Namespace) -> int: discovery = evidence_module.discover(repo_root, build_dir) manifests: list[dict] = [] - siblings: dict[str, dict] = {} - for wheel in build_module.in_dependency_order( - sorted(Path(item) for item in args.wheel) - ): + wheels = sorted(Path(item) for item in args.wheel) + # What every wheel of this run says about itself, before any is described. + # A document reference needs the referenced document to exist; these facts + # do not, so a requirement on a sibling is never an install-time unknown. + siblings = build_module.siblings_of(wheels, repo_root) + for wheel in build_module.in_dependency_order(wheels): manifest = build_module.build( repo_root, build_dir, @@ -86,6 +88,7 @@ def _cmd_build(args: argparse.Namespace) -> int: manifests.append(manifest) described = manifest["wheels"][0]["describes"] siblings[canonicalize_name(described["name"])] = described + siblings[canonicalize_name(described["name"])]["wheel"] = wheel.name print(f"packaged evidence into {wheel.name}") # Through the same merge as a release: inheriting the first wheel's manifest diff --git a/scripts/sbom/sbom/document.py b/scripts/sbom/sbom/document.py index 72b4e36acf..1414c3c84f 100644 --- a/scripts/sbom/sbom/document.py +++ b/scripts/sbom/sbom/document.py @@ -208,6 +208,34 @@ def _dependency_refs( return refs +def _source_info(project: dict, evidence_doc: dict, build: dict, wheel) -> str: + """Where this wheel came from, stating only what applies to it. + + One build produces several wheels, so its architecture and configuration are + facts about the compilation -- which a pure-python wheel did not have. A + placeholder rendered as a noun ("Built from NOASSERTION") is worse than an + omitted clause. + """ + source = evidence_doc.get("project", {}) + parts = [] + if project["homepage"] != NOASSERTION: + parts.append(f"Built from {project['homepage']}") + commit = source.get("commit") + if commit: + described = source.get("describe") or "" + parts.append( + f"at commit {commit}" + # A tree with uncommitted changes did not produce what that commit + # produces, and a report nothing can reproduce has to say so. + + (" with uncommitted changes" if described.endswith("-dirty") else "") + ) + # `none-any` is the tag for a wheel with nothing compiled in it. + if "none-any" not in wheel.path.name: + parts.append(f"arch {build.get('arch')}") + parts.append(f"CMAKE_BUILD_TYPE {build.get('config')}") + return "; ".join(parts) + "." if parts else NOASSERTION + + def _wheel_conclusion(declared: str, inventory: Inventory) -> str: """The declared licence, AND the terms of everything redistributed under its own. @@ -346,11 +374,7 @@ def build_document( } ], "hasFiles": [file_ids[name] for name in sorted(file_ids)], - "sourceInfo": ( - f"Built from {project['homepage']} at commit " - f"{evidence_doc.get('project', {}).get('commit') or NOASSERTION}; " - f"arch {build.get('arch')}; CMAKE_BUILD_TYPE {build.get('config')}." - ), + "sourceInfo": _source_info(project, evidence_doc, build, wheel), } packages = [wheel_package] @@ -455,24 +479,24 @@ def build_document( sibling = (siblings or {}).get(canonicalize_name(name)) if sibling is None: element = package_id + comment = ( + f"{declared} Resolved at install time; this document states no " + "license for it." + ) packages.append( _unanalyzed_package( - package_id, - name, - f"{declared} Resolved at install time; this document states " - "no license for it.", - _dependency_refs(name, group[0]), + package_id, name, comment, _dependency_refs(name, group[0]) ) ) - else: - # Built by this same run, so it is not an install-time unknown: the - # dependency points straight at the package in that wheel's own - # document. A local stub beside it would state the version, supplier - # and licence a second time, from the same source. - element = f"DocumentRef-{licensing.spdx_safe(sibling['filename'])}" + elif sibling.get("namespace"): + # Built by this same run and already described, so the dependency + # points straight at the package in that wheel's own document. A + # local stub beside it would restate version, supplier and licence + # from the same source. + document_ref = f"DocumentRef-{licensing.spdx_safe(sibling['filename'])}" external_documents.append( { - "externalDocumentId": element, + "externalDocumentId": document_ref, "spdxDocument": sibling["namespace"], "checksum": { "algorithm": "SHA1", @@ -480,13 +504,39 @@ def build_document( }, } ) - element = f"{element}:{WHEEL_PACKAGE_ID}" - - comment = declared + ( - f" Built by this same build as {sibling['name']} {sibling['version']}." - if sibling - else "" - ) + element = f"{document_ref}:{WHEEL_PACKAGE_ID}" + comment = ( + f"{declared} Built by this same build as {sibling['name']} " + f"{sibling['version']}; described by {sibling['filename']}." + ) + else: + # Built by this same run but not yet described: two wheels can + # require each other, and a document citing another's digest cannot + # be written before it. What the wheel states about itself needs no + # document, so none of it is left unasserted. + element = package_id + comment = ( + f"{declared} Built by this same build; its own document " + f"describes it, and is not referenced here because the two " + f"wheels require each other." + ) + packages.append( + { + "SPDXID": package_id, + "name": name, + "versionInfo": sibling["version"], + "supplier": sibling["supplier"], + "downloadLocation": NOASSERTION, + "filesAnalyzed": False, + "licenseConcluded": NOASSERTION, + "licenseDeclared": sibling["license_declared"], + "copyrightText": NOASSERTION, + "externalRefs": _dependency_refs( + name, group[0], version=sibling["version"] + ), + "comment": comment, + } + ) # Optional only if every line that asks for it is gated on an extra. One # unconditional line makes the dependency unconditional. if all(_is_extra_gated(item) for item in group): diff --git a/scripts/sbom/sbom/licensing.py b/scripts/sbom/sbom/licensing.py index 1c181f238a..aa624d48e3 100644 --- a/scripts/sbom/sbom/licensing.py +++ b/scripts/sbom/sbom/licensing.py @@ -176,13 +176,13 @@ class CorpusError(Exception): _CORPUS: tuple[str, dict[str, Reference]] | None = None -_CORPUS_META: dict[str, str] = {} +_CORPUS_META: dict[str, object] = {} # Parsing and n-gramming the whole SPDX list costs about half a second. Keyed by # directory and only ever read within one process, so a re-load is free. _PARSED: dict[str, tuple[str, dict[str, Reference]]] = {} -def corpus_provenance() -> dict[str, str]: +def corpus_provenance() -> dict[str, object]: """Which reference texts an identification was made against.""" return dict(_CORPUS_META) @@ -238,7 +238,7 @@ def load_corpus(json_dir: Path) -> str: "license_list_version": version, "source": str(json_dir), "commit": _checkout_commit(json_dir.parent), - "licenses": str(len(entries)), + "licenses": len(entries), } ) return version diff --git a/scripts/sbom/sbom/report.py b/scripts/sbom/sbom/report.py index 8e514bcf19..5d5124d7ab 100644 --- a/scripts/sbom/sbom/report.py +++ b/scripts/sbom/sbom/report.py @@ -191,6 +191,11 @@ def report(inventory: Inventory, evidence_doc: dict, wheel_name: str) -> dict: "schema": "isaaccapture-license-report/2", "wheel": wheel_name, "generated_from_commit": evidence_doc.get("project", {}).get("commit"), + # The commit alone does not describe a tree that was modified, and this + # report is what a reader uses to reproduce the build. + "source_tree_modified": str( + evidence_doc.get("project", {}).get("describe") or "" + ).endswith("-dirty"), "spdx_license_list_version": evidence_doc.get("license_list_version"), "license_data": evidence_doc.get("license_data", {}), "components": components, @@ -213,7 +218,13 @@ def report_markdown(payload: dict) -> str: lines = [ f"# Component-to-license report — `{payload['wheel']}`", "", - f"Source commit: `{payload['generated_from_commit'] or 'unknown'}` ", + f"Source commit: `{payload['generated_from_commit'] or 'unknown'}`" + + ( + " (built from a tree with uncommitted changes)" + if payload.get("source_tree_modified") + else "" + ) + + " ", f"SPDX license list: `{payload['spdx_license_list_version'] or 'unknown'}` " f"(spdx/license-list-data `{payload.get('license_data', {}).get('commit', '?')[:12]}`)", "", @@ -256,6 +267,17 @@ def report_markdown(payload: dict) -> str: else: lines.append("None recorded.") + lines += ["", "## License texts packaged in this wheel", ""] + lines += [ + f"- `{item}`" for item in payload.get("packaged_license_files") or [] + ] or ["None."] + if payload.get("packaged_license_files"): + lines += [ + "", + "Declared in METADATA as `License-File`: " + + ("yes" if payload.get("metadata_license_files_declared") else "no"), + ] + lines += ["", "## Consumer requirements (wheel metadata)", ""] lines += [f"- `{item}`" for item in payload["consumer_requirements"]] or [ "None declared." diff --git a/scripts/sbom/sbom/validate.py b/scripts/sbom/sbom/validate.py index 0d7b602d29..fc374aec5d 100644 --- a/scripts/sbom/sbom/validate.py +++ b/scripts/sbom/sbom/validate.py @@ -570,10 +570,78 @@ def _check_manifest( return failures +def _cross_document_refs( + spdx: dict, documents: dict[str, bytes], filename: str +) -> list[str]: + """An external document reference has to resolve to a document in this set. + + Declaring one is not the check: the digest is what binds it, and a wheel + rebuilt without its sibling would otherwise carry a stale one and pass. + """ + failures: list[str] = [] + by_namespace = { + json.loads(body).get("documentNamespace"): (name, body) + for name, body in documents.items() + } + for ref in spdx.get("externalDocumentRefs", []): + found = by_namespace.get(ref.get("spdxDocument")) + if found is None: + failures.append( + f"{filename}: {ref.get('externalDocumentId')} names a document " + f"this release does not publish ({ref.get('spdxDocument')})" + ) + continue + name, body = found + stated = (ref.get("checksum") or {}).get("checksumValue") + actual = hashlib.sha1(body).hexdigest() # noqa: S324 - SPDX 2.3 says SHA1 + if stated != actual: + failures.append( + f"{filename}: {ref.get('externalDocumentId')} states sha1 {stated} " + f"for {name}, which hashes to {actual}" + ) + # And the elements those references point at have to be in that document. + ids = { + ref["externalDocumentId"]: json.loads(by_namespace[ref["spdxDocument"]][1]) + for ref in spdx.get("externalDocumentRefs", []) + if ref.get("spdxDocument") in by_namespace + } + for relationship in spdx.get("relationships", []): + for side in ("spdxElementId", "relatedSpdxElement"): + prefix, _, element = str(relationship[side]).partition(":") + target = ids.get(prefix) + if target is None or not element: + continue + known = {target["SPDXID"]} + known |= {item["SPDXID"] for item in target.get("packages", [])} + known |= {item["SPDXID"] for item in target.get("files", [])} + if element not in known: + failures.append( + f"{filename}: {relationship[side]} names an element the " + "referenced document does not define" + ) + return failures + + def check_set(manifest_path: Path, wheel_dir: Path, evidence_dir: Path) -> list[str]: """Run :func:`check` over every wheel a merged manifest advertises.""" failures: list[str] = [] manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + + # The whole release is here, which is the only place a reference from one + # document to another can be resolved rather than merely declared. + published: dict[str, bytes] = {} + for entry in manifest.get("wheels") or []: + record = _record_for(manifest, entry.get("filename") or "") + sidecar = _sidecar(evidence_dir, record, "spdx") if record else None + if sidecar and sidecar.is_file(): + published[sidecar.name] = sidecar.read_bytes() + + for name, body in sorted(published.items()): + try: + failures.extend(_cross_document_refs(json.loads(body), published, name)) + except json.JSONDecodeError as error: + failures.append(f"{name}: not readable as a document ({error})") + for entry in manifest.get("wheels") or []: named = entry.get("filename") if not named: diff --git a/tests/python/sbom/synth.py b/tests/python/sbom/synth.py index 80c004a7ee..3884bcfbda 100644 --- a/tests/python/sbom/synth.py +++ b/tests/python/sbom/synth.py @@ -527,6 +527,9 @@ def _file_api_reply(root: Path, build: Path, extension: str) -> None: "License-Expression: Apache-2.0", "License-File: LICENSE.md", "Requires-Python: >=3.11", + # The transition wheel requires this one and this one requires it back; + # a cycle is the shape the real pair has. + "Requires-Dist: isaacteleop>=0.4.0", "Requires-Dist: numpy>=1.23.0", "Requires-Dist: pyyaml>=6.0.3", "", diff --git a/tests/python/sbom/test_wheel_sbom.py b/tests/python/sbom/test_wheel_sbom.py index 64392bd07f..5309a141c8 100644 --- a/tests/python/sbom/test_wheel_sbom.py +++ b/tests/python/sbom/test_wheel_sbom.py @@ -319,6 +319,7 @@ def test_a_requirement_this_run_also_built_points_at_its_own_document( "Version: 0.4.0", "License-Expression: Apache-2.0", "Requires-Dist: isaaccapture>=0.4.0", + "Author: Example Org", "", ] ).encode(), @@ -352,6 +353,21 @@ def test_a_requirement_this_run_also_built_points_at_its_own_document( == 0 ) + # The two require each other, so only one document can cite the other's + # digest; neither may report the other as an install-time unknown. + sibling_doc = json.loads( + (out / f"{WHEEL_NAME.removesuffix('.whl')}.spdx.json").read_text() + ) + back = next( + package + for package in sibling_doc["packages"] + if package["name"] == "isaacteleop" + ) + assert back["versionInfo"] == "0.4.0" + assert back["licenseDeclared"] == "Apache-2.0" + assert back["supplier"] != "NOASSERTION" + assert "require each other" in back["comment"] + spdx = json.loads((out / "isaacteleop-0.4.0-py3-none-any.spdx.json").read_text()) assert not [ package From 47b296a04785aa2858d1f80109e38dd8963fe495 Mon Sep 17 00:00:00 2001 From: Andrei Aristarkhov Date: Thu, 1 Oct 2026 03:20:51 -0700 Subject: [PATCH 057/101] build(sbom): conclude every licence the wheel carries, and find every holder `licenseConcluded` on the wheel listed only the components the build could not name, under the reasoning that the declaration covers the rest. It does not: Apache-2.0 covers neither MIT, nor Zlib, nor BSD-3-Clause, nor Qhull, nor BSL-1.0, nor the CC-BY-4.0 the same package asserted from its own files. Every component's terms are in it now, combined by identifier so a compound one does not arrive nested in parentheses and an id two components share appears once. Three more ways a holder went missing, each against a packaged file. A notice whose holder sits on the next line with no comma to join them: Debian copyright files write the years alone and indent the holder below, so libbsd named neither the Regents of the University of California, nor the NetBSD Foundation, nor Todd C. Miller, and qhull named no one at all. A notice with a long lead-in: the cap was forty characters and `Box collision code (engine_collision_box.c) is Copyright 2016 Svetoslav Kolev` needs forty-six. And a clause broken by the end of its line left "Copyright (c) 2007-2010 by Baptiste Lepilleur, and is" standing as a notice. libmd's terms are an aggregate of nine stanzas; two generic BSD variants matched at high containment and together covered half the file, and the document said the identifiers were merely newer than its grammar. It now says what is true -- that what matched accounts for 54% of the text and names part of it, not the whole. The matches themselves still stand: low coverage is not evidence of a wrong identification, as glm's dual-licence file and ccd's BSD text both show. VERS orders constraints by version, so `<1.9` no longer sorts before `>=1.6`. Signed-off-by: Andrei Aristarkhov --- scripts/sbom/sbom/document.py | 61 ++++++++++++------------ scripts/sbom/sbom/licensing.py | 84 ++++++++++++++++++++++++++++------ scripts/sbom/sbom/report.py | 7 +-- 3 files changed, 104 insertions(+), 48 deletions(-) diff --git a/scripts/sbom/sbom/document.py b/scripts/sbom/sbom/document.py index 1414c3c84f..157a01878e 100644 --- a/scripts/sbom/sbom/document.py +++ b/scripts/sbom/sbom/document.py @@ -11,6 +11,7 @@ from packageurl import PackageURL from packaging.requirements import Requirement from packaging.utils import canonicalize_name +from packaging.version import InvalidVersion, Version from . import SPDX_VERSION, TOOL_NAME, TOOL_VERSION, licensing, report, stamped_now from .discovery import Component @@ -130,14 +131,7 @@ def _extracted_entries(components: list[Component]) -> list[dict]: "_for": component.name, "extractedText": body, "comment": ( - ( - "Matched no text in the SPDX reference corpus" - if not any(item.matches for item in grants) - # Matched, but under an id the grammar shipped with this - # document cannot validate, so it cannot be named here. - else "Matched an identifier newer than the SPDX grammar " - "this document ships with" - ) + licensing.why_unnamed(grants) + f"; reproduced here verbatim from {shipped}." ), } @@ -176,6 +170,14 @@ def _is_extra_gated(requirement: Requirement) -> bool: ) +def _by_version(specifier): + """Specifier clauses in version order, or as given if one will not parse.""" + try: + return sorted(specifier, key=lambda item: Version(item.version)) + except InvalidVersion: + return list(specifier) + + def _dependency_refs( name: str, requirement: Requirement, version: str | None = None ) -> list[dict]: @@ -194,9 +196,11 @@ def _dependency_refs( ).to_string(), } ] - constraints = sorted( - f"{item.operator}{item.version}" for item in requirement.specifier - ) + # VERS orders constraints by version, not by the text of the operator: the + # string sort put `<1.9` before `>=1.6` and a strict parser rejects that. + constraints = [ + f"{item.operator}{item.version}" for item in _by_version(requirement.specifier) + ] if constraints: refs.append( { @@ -237,25 +241,26 @@ def _source_info(project: dict, evidence_doc: dict, build: dict, wheel) -> str: def _wheel_conclusion(declared: str, inventory: Inventory) -> str: - """The declared licence, AND the terms of everything redistributed under its own. + """The declared licence, AND the terms of everything it redistributes. - A component whose terms the build could not name travels as a LicenseRef, - and those are exactly the ones a reader must not miss -- a proprietary SDK - EULA among them. Components under a licence the declaration already covers - add nothing and are left out. + Every component, not only the ones the build could not name: Apache-2.0 does + not cover MIT, Zlib, BSD-3-Clause, Qhull or BSL-1.0, so leaving those out + said the wheel carries no code under them. Only a component concluding the + declared licence itself adds nothing. """ - others = sorted( - { - component.license_concluded - for component in inventory.components_present.values() - if "LicenseRef-" in component.license_concluded - } - ) - if not others: - return declared - if declared == NOASSERTION: - return licensing.combine(others) - return licensing.combine([declared, *others]) + # By identifier, not by expression: combining whole expressions nests the + # compound ones in parentheses and repeats an id two components share. + identifiers: set[str] = set() + for component in inventory.components_present.values(): + if component.license_concluded == NOASSERTION: + continue + try: + identifiers.update(licensing.identifiers_in(component.license_concluded)) + except licensing.ExpressionReadError: + identifiers.add(component.license_concluded) + if declared != NOASSERTION: + identifiers.add(declared) + return licensing.combine(sorted(identifiers)) if identifiers else declared def build_document( diff --git a/scripts/sbom/sbom/licensing.py b/scripts/sbom/sbom/licensing.py index aa624d48e3..994700c981 100644 --- a/scripts/sbom/sbom/licensing.py +++ b/scripts/sbom/sbom/licensing.py @@ -46,6 +46,12 @@ # genuine match sits well above it -- a Debian copyright file naming several # licenses around the one matched is the low end, at a quarter. _COVERAGE_THRESHOLD = 0.20 +# ...and the matches together have to account for most of the file before the +# set of them is called an identification. A Debian copyright file aggregates +# many stanzas, and generic BSD boilerplate clears containment against variants +# the file never mentions; two such matches covering half the text name the file +# no better than its own words do. +_EXPLAINED_THRESHOLD = 0.75 # Two licenses are one family when either text is nearly inside the other. _FAMILY_THRESHOLD = 0.90 # REUSE-IgnoreStart @@ -478,6 +484,35 @@ def expressible(license_id: str) -> bool: return False +def explains_the_text(matches) -> bool: + """Whether the matched licences account for most of what the file says. + + Not a test of whether an identification is right -- a dual-licence file + matches correctly and scores low -- only of whether it explains the file, + which is what the document has to say when it cannot name one. + """ + return sum(coverage for _, _, coverage in matches) >= _EXPLAINED_THRESHOLD + + +def why_unnamed(grants: list[LicenseEvidence]) -> str: + """Why terms travel verbatim rather than under an SPDX identifier.""" + if not any(item.matches for item in grants): + return "Matched no text in the SPDX reference corpus" + partial = [item for item in grants if not explains_the_text(item.matches)] + if partial: + share = max( + sum(coverage for _, _, coverage in item.matches) for item in partial + ) + return ( + "What matched in the SPDX reference corpus accounts for only " + f"{share:.0%} of this text, so the matches name part of it and not " + "the whole" + ) + # Matched, but under an id the grammar shipped with this document cannot + # validate, so it cannot be named here. + return "Matched an identifier newer than the SPDX grammar this document ships with" + + def expression( evidence: list[LicenseEvidence], component_key: str | None = None ) -> tuple[str, str]: @@ -586,12 +621,16 @@ def fold_notices(notices) -> list[str]: # A notice is `Copyright` plus a marker that prose does not carry: a (c), a ©, -# or a year. It need not start the line -- qhull writes `Qhull, Copyright (c) -# 1993-2020` -- so a short lead-in is allowed before it. +# or a year. It need not start the line and the lead-in is not bounded: a NOTICE +# writes `Box collision code (engine_collision_box.c) is Copyright 2016 ...`. _PLAIN_COPYRIGHT = re.compile( - r"^[^\n\r]{0,40}?(?PCopyright\b\s*(?:\(c\)|©|\d{4})[^\n\r]*)$", + r"^.*?(?PCopyright\b\s*(?:\(c\)|©|\d{4})[^\n\r]*)$", re.MULTILINE | re.IGNORECASE, ) +# A clause broken by the end of a line, left dangling by taking the line alone. +_DANGLING = re.compile(r"(?:[,;]?\s+(?:and|or|is|was|are|were|by|under|the|a))+$", re.I) +# Years, year ranges and the markers around them: what is left is the holder. +_YEARS_AND_MARKS = re.compile(r"\(c\)|©|\d{4}(?:\s*-\s*\d{2,4})?|[,;]", re.I) # Lines of a licence body that mention copyright without stating one. _NOT_A_NOTICE = re.compile( r"\bshall be\b|\bowner or entity\b|\bbe liable\b|\bmeans?\b|" @@ -653,26 +692,43 @@ def read_notices(text: str) -> list[str]: if not match: continue notice = " ".join(match.group("notice").split()).rstrip("*/ ") - # A holder can run on across lines -- libccd names a department, a - # faculty and a university -- and a trailing comma is the author saying - # so. Anything else ends the notice. + # The holder can sit on the lines below: after a trailing comma (libccd + # names a department, a faculty and a university), or after nothing at + # all (Debian copyright files and qhull put the years on one line and the + # holder on the next). Keep reading until a holder appears. step = index - while notice.endswith(",") and step + 1 < len(lines): - step += 1 - notice = f"{notice} {' '.join(lines[step].split())}".strip().rstrip("*/ ") + while step + 1 < len(lines) and step - index < 3: + if not (notice.endswith(",") or not _names_a_holder(notice)): + break + ahead = step + 1 + # One blank line may sit between the years and the holder; qhull + # writes `Qhull, Copyright (c) 1993-2020`, a blank, then the holder. + if not lines[ahead].strip() and not _names_a_holder(notice): + ahead += 1 + if ahead >= len(lines): + break + following = " ".join(lines[ahead].split()) + # A new notice, or a field of its own, is not this one's holder. + if not following or _PLAIN_COPYRIGHT.match(following) or ":" in following: + break + step = ahead + notice = f"{notice} {following}".strip().rstrip("*/ ") + notice = _DANGLING.sub("", notice).rstrip("*/,; ") if _PLACEHOLDER.search(notice) or _NOT_A_NOTICE.search(notice): continue - # Something has to follow the word: a holder, or at least the years the - # file states. qhull writes `Qhull, Copyright (c) 1993-2020` and puts the - # holder on the lines below, so a year alone is still what it states. - rest = notice[len("Copyright") :] - if not re.search(r"[A-Za-z]{2}", rest) and not re.search(r"\d{4}", rest): + if not _names_a_holder(notice) and not re.search(r"\d{4}", notice): continue if notice not in found: found.append(notice) return found +def _names_a_holder(notice: str) -> bool: + """Whether anything is left once the years and their punctuation come out.""" + rest = _YEARS_AND_MARKS.sub(" ", notice[len("Copyright") :]) + return bool(re.search(r"[A-Za-z]{2}", rest)) + + def read_notice(text: str) -> str | None: """The first notice a licence text states, where only one is wanted.""" notices = read_notices(text) diff --git a/scripts/sbom/sbom/report.py b/scripts/sbom/sbom/report.py index 5d5124d7ab..c32dd5d5a9 100644 --- a/scripts/sbom/sbom/report.py +++ b/scripts/sbom/sbom/report.py @@ -176,12 +176,7 @@ def report(inventory: Inventory, evidence_doc: dict, wheel_name: str) -> dict: "component": key, "reason": ( "the shipped grant is carried verbatim rather than named: " - + ( - "it matches no text in the SPDX reference corpus" - if not any(item.identified for item in grants) - else "what it matches cannot be expressed in the SPDX " - "grammar shipped with this document" - ) + + licensing.why_unnamed(grants).lower() ), "evidence": [item.path for item in grants], } From 6d3ecc3bb367a017215bd3c1e7e5a49613b5825f Mon Sep 17 00:00:00 2001 From: Andrei Aristarkhov Date: Thu, 1 Oct 2026 03:42:01 -0700 Subject: [PATCH 058/101] docs(sbom): stop the design doc contradicting what the collector does Three claims in it were false of this build's own output, all on the behaviour that changed: that a reference text is never substituted, that the texts under `.dist-info/licenses/LICENSES/` are this repository's REUSE pool, and that a component shipping no licence text blocks publication. A component stating its licence only in its source headers is the case the collector handles by packaging the reference text and saying so; the doc told a reader the opposite, which matters because a template and an author's own notice look alike. Alongside, what the audit found in the artifacts themselves: libmd credits Poul-Henning Kamp, Colin Plumb, Todd C. Miller and Steve Reid through Debian's `Copyright:` field, which may hold a bare name -- no `Copyright` token, so a parser looking for the word found nobody. They are the authors of the md5, rmd160 and sha1 sources inside the library this wheel ships. The field also holds prose and the literal `None`, neither of which is a holder. A file's own `copyrightText` kept only the first notice while the package roll-up kept them all, so the per-file field gave one of glfw's two holders and one of libbsd's forty-six. Both artifacts a person reads now say when a packaged text is the SPDX reference substituted for a component that ships none; the notices file indents a second holder, which a lazy Markdown continuation was rendering into the first; a component checked into this repository is no longer excluded as "fetched for this build"; and an archive's member count counts what it says it counts. Signed-off-by: Andrei Aristarkhov --- docs/source/references/sbom.rst | 21 +++++++++++------- scripts/sbom/sbom/build.py | 9 +++++++- scripts/sbom/sbom/evidence.py | 2 +- scripts/sbom/sbom/inventory.py | 9 +++++++- scripts/sbom/sbom/licensing.py | 38 ++++++++++++++++++++++++++++++++- scripts/sbom/sbom/report.py | 24 ++++++++++++++++++--- 6 files changed, 88 insertions(+), 15 deletions(-) diff --git a/docs/source/references/sbom.rst b/docs/source/references/sbom.rst index 4371bccb55..8c93193354 100644 --- a/docs/source/references/sbom.rst +++ b/docs/source/references/sbom.rst @@ -202,9 +202,10 @@ verbatim, copyright holders and all. Substituting a reference text would convey and drop the notice — for MIT and the BSD licenses those are the same file, differing by the one line that names who licensed it to you — and it would replace the component's actual grant with our reading of it, which is what the matching is for and not what it -decides. Where a component has *no* license file, and only states an identifier in its -source headers, there is nothing of its own to package: the text that identifier names in -the pinned list ships instead, together with the copyright line those headers state. +decides. So a reference text is substituted only where there is nothing of the component's +own to package: it states an identifier in its source headers and ships no licence file. +The document says so for every such text, in as many words, because a reader cannot tell a +template from an author's own notice by looking at it. A component can also state terms for one part of itself — a vendored asset set inside a larger project, with its own ``LICENSE`` and its own holder. Those are read and packaged @@ -224,9 +225,10 @@ this artifact" is told about the proprietary payload rather than about the decla Never a file from this checkout. ``LICENSES/`` holds this repository's REUSE pool, whose copies exist to license *this* project and carry its notices; packaging one as a third -party's terms would put our copyright above their code. Those texts ship under -``.dist-info/licenses/LICENSES/`` as the project's own, and have nothing to do with the -matching or with any other component. +party's terms would put our copyright above their code. A substituted text comes from the +pinned list, never from the pool — and under ``.dist-info/licenses/LICENSES/`` a wheel +carries whichever applies: the project's own pool files where it has them, and otherwise +the reference text for the licence it declares. A match needs a canonical license text to be almost entirely present *and* to account for a real share of the file: a tenth of a long agreement reproducing a license in an appendix is @@ -241,8 +243,11 @@ travels in the document, which says "these are the terms that shipped" without c know which license they are. Those components are listed in the report under *License evidence gaps*. -A component that ships **no** license text at all blocks publication, because packaging the -text is the obligation the wheel has to meet. +A component that states a licence nowhere at all blocks publication, because packaging the +text is the obligation the wheel has to meet. Stating one in its source headers and nowhere +else is not that case: there the reference text for the identifier those headers name ships +instead, labelled in the document as a substitution rather than as the component's own words +— it conveys the terms and names no holder, so the headers' copyright line travels with it. A shared library that none of those indexes explains did not come from this build, so the last thing asked is the build machine itself: resolve the library's SONAME, and take its diff --git a/scripts/sbom/sbom/build.py b/scripts/sbom/sbom/build.py index e02edb9f60..9949663f01 100644 --- a/scripts/sbom/sbom/build.py +++ b/scripts/sbom/sbom/build.py @@ -403,7 +403,14 @@ def build( if name in additions else "distribution metadata, updated to declare the packaged license files", spdx_tag=licensing.normalized_tag(body) if packaged_text else None, - copyright_text=licensing.read_notice(body) if packaged_text else None, + # Every holder the text names, as the package roll-up does: keeping + # the first makes a per-file notice that does not discharge the + # obligation -- libbsd's copyright file names forty-six. + copyright_text="\n".join( + licensing.fold_notices(licensing.read_notices(body)) + ) + if packaged_text + else None, ) # The document is added below, after it has been rendered; record it now so # the evidence accounts for every member of the finished wheel. diff --git a/scripts/sbom/sbom/evidence.py b/scripts/sbom/sbom/evidence.py index 4785cc0c22..4b4a5a29ee 100644 --- a/scripts/sbom/sbom/evidence.py +++ b/scripts/sbom/sbom/evidence.py @@ -159,7 +159,7 @@ def _archive_components( download_location=source_url or "NOASSERTION", source_info=( f"Unpacked from {display} (sha256:{record['sha256']}, " - f"{record['member_count']} members), {held}." + f"{record['member_count']} files), {held}." ), license_concluded=concluded, license_declared=declared, diff --git a/scripts/sbom/sbom/inventory.py b/scripts/sbom/sbom/inventory.py index a97a180551..24ae996bd3 100644 --- a/scripts/sbom/sbom/inventory.py +++ b/scripts/sbom/sbom/inventory.py @@ -620,8 +620,15 @@ def build( for key in sorted(roles) if key in resolver.components } + # Say how the component got here. Checked-in sources were never fetched, and + # the collector's own vocabulary for them elsewhere says so. absent = { - key: "fetched for this build; contributes to no file in this wheel" + key: ( + "checked into this repository" + if resolver.components[key].kind == "vendored-source" + else "fetched for this build" + ) + + "; contributes to no file in this wheel" for key in sorted(resolver.components) if key not in roles } diff --git a/scripts/sbom/sbom/licensing.py b/scripts/sbom/sbom/licensing.py index 994700c981..f7193cde15 100644 --- a/scripts/sbom/sbom/licensing.py +++ b/scripts/sbom/sbom/licensing.py @@ -630,6 +630,12 @@ def fold_notices(notices) -> list[str]: # A clause broken by the end of a line, left dangling by taking the line alone. _DANGLING = re.compile(r"(?:[,;]?\s+(?:and|or|is|was|are|were|by|under|the|a))+$", re.I) # Years, year ranges and the markers around them: what is left is the holder. +# A sentence, as against a name: the Debian `Copyright:` field holds both. +_PROSE = re.compile( + r"\b(derived|contributed|reserved|portions?|redistribut\w+|written|" + r"modified|see|provided|this|these|some|all rights)\b", + re.IGNORECASE, +) _YEARS_AND_MARKS = re.compile(r"\(c\)|©|\d{4}(?:\s*-\s*\d{2,4})?|[,;]", re.I) # Lines of a licence body that mention copyright without stating one. _NOT_A_NOTICE = re.compile( @@ -686,7 +692,7 @@ def read_notices(text: str) -> list[str]: `Copyright [yyyy] [name of copyright owner]` -- which names nobody. """ lines = text.splitlines() - found: list[str] = [] + found: list[str] = _debian_copyright_fields(lines) for index, line in enumerate(lines): match = _PLAIN_COPYRIGHT.match(line) if not match: @@ -723,6 +729,36 @@ def read_notices(text: str) -> list[str]: return found +def _debian_copyright_fields(lines: list[str]) -> list[str]: + """Holders a Debian copyright file states without the word `Copyright`. + + `Copyright:` is a field there, and its value may be a bare name -- libmd + credits Poul-Henning Kamp, Colin Plumb and Steve Reid that way. Nothing else + in the file names them, so a parser looking for the word finds nobody. + """ + found: list[str] = [] + for index, line in enumerate(lines): + if line.rstrip() != "Copyright:": + continue + for position, value in enumerate(lines[index + 1 :]): + if not value[:1].isspace() or not value.strip(): + break + holder = " ".join(value.split()).rstrip(".") + # A field whose value opens with a notice proper is read by the scan + # below, prose and all; only a field naming a bare holder is read + # here, and only for as long as it keeps naming one. + if _PLAIN_COPYRIGHT.match(holder) or _PROSE.search(holder): + break + # Debian writes `None` where a package states no holder. + if holder.lower() in {"none", "n/a", "unknown"}: + break + if position and len(holder) > 60: + break + if _names_a_holder(f"Copyright {holder}") and holder not in found: + found.append(holder) + return found + + def _names_a_holder(notice: str) -> bool: """Whether anything is left once the years and their punctuation come out.""" rest = _YEARS_AND_MARKS.sub(" ", notice[len("Copyright") :]) diff --git a/scripts/sbom/sbom/report.py b/scripts/sbom/sbom/report.py index c32dd5d5a9..95a75f69fd 100644 --- a/scripts/sbom/sbom/report.py +++ b/scripts/sbom/sbom/report.py @@ -112,8 +112,12 @@ def notices_markdown(inventory: Inventory, dist_info: str, wheel_name: str) -> s lines.append(f"- License: `{component.license_concluded}`") if component.copyright_text != "NOASSERTION": # A reference text names no holder, so for a component that ships no - # licence file of its own this is the only notice in the wheel. - lines.append(f"- Copyright: {component.copyright_text}") + # licence file of its own this is the only notice in the wheel. One + # per line, indented: a bare newline inside a list item is a lazy + # continuation and renders several notices as one paragraph. + notices = component.copyright_text.splitlines() + lines.append(f"- Copyright: {notices[0]}") + lines.extend(f" {item}" for item in notices[1:]) if component.license_declared != component.license_concluded: lines.append(f"- Declared: `{component.license_declared}`") if component.homepage != "NOASSERTION": @@ -122,7 +126,16 @@ def notices_markdown(inventory: Inventory, dist_info: str, wheel_name: str) -> s lines.append(f"- Version: `{component.version}`") lines.append(f"- Reaches this wheel because {_roles(inventory, key)}") for item in component.evidence: - lines.append(f"- Text: `{packaged_license_path(dist_info, key, item)}`") + packaged = packaged_license_path(dist_info, key, item) + lines.append( + f"- Text: `{packaged}`" + + ( + " — the SPDX reference text for this licence, not this " + "component's own file, which it ships none of" + if item.origin == licensing.SPDX_REFERENCE + else "" + ) + ) if not component.evidence: lines.append("- No license text was found in this component's own files.") lines.append("") @@ -236,6 +249,11 @@ def report_markdown(payload: dict) -> str: "
".join( f"`{record['path']}`" + (f" → {record['identified']}" if record["identified"] else "") + + ( + " (SPDX reference text, substituted)" + if record["origin"] == licensing.SPDX_REFERENCE + else "" + ) for record in item["evidence"] ) or "—" From d984b1562c6ab54d8c85e1bc38c5be859add31e2 Mon Sep 17 00:00:00 2001 From: Andrei Aristarkhov Date: Thu, 1 Oct 2026 03:56:03 -0700 Subject: [PATCH 059/101] build(sbom): read the commoner Debian copyright field, and break the lines `Copyright: Jane Doe` is how most Debian copyright files name a holder, and the field reader took only the indented form. On this host 1528 of 1620 such files use the same-line spelling; on a sample of forty, three quarters yielded no holder at all. libbsd and libmd happen to use the other form, so these wheels were unaffected -- the next system library auditwheel vendors would not have been. Prose and the literal `None` are still not holders. The notices file joined multiple holders with a newline, then with an indented newline. Both are lazy continuations in Markdown, so a renderer ran distinct legal notices together -- libmd's three individuals read as one string. They are joined with an explicit break now, checked against a renderer rather than reasoned about. Two claims in the design doc survived the last pass, in places the sweep missed: the directory diagram still called every packaged text the component's own file, and the maintenance section still said a component shipping no licence text stops the build. Its heading for excluded components no longer says "fetched" over rows that were checked in. Signed-off-by: Andrei Aristarkhov --- docs/source/references/sbom.rst | 9 +++-- scripts/sbom/sbom/licensing.py | 9 ++++- scripts/sbom/sbom/report.py | 15 ++++---- tests/python/sbom/test_licensing.py | 59 +++++++++++++++++++++++++++++ 4 files changed, 79 insertions(+), 13 deletions(-) diff --git a/docs/source/references/sbom.rst b/docs/source/references/sbom.rst index 8c93193354..378d11d438 100644 --- a/docs/source/references/sbom.rst +++ b/docs/source/references/sbom.rst @@ -20,7 +20,7 @@ What a wheel carries │ ├── LICENSE.md # this project's own license │ ├── LICENSES/*.txt # texts the SPDX headers on this project's sources name │ ├── THIRD-PARTY-NOTICES.md # component -> license -> text, with roles - │ └── third-party// # each redistributed component's own file, verbatim + │ └── third-party// # its own file verbatim, or a reference text, labelled └── sboms/ └── .spdx.json # SPDX 2.3 contents inventory @@ -128,7 +128,7 @@ packaged assets, for all six published variants (``x86_64`` and ``aarch64`` × C Deliberately **not** described by it: - Build-time-only inputs — test frameworks, code generators, CMake modules, toolchains. - They appear in the report under *Fetched for this build but not in this wheel*; a full + They appear in the report under *Present in this build but not in this wheel*; a full build inventory is separate work. - Source distributions and the ``pip install .`` path, which produce different contents. - Containers and the web client, which are distributed on their own. @@ -292,8 +292,9 @@ inventory with its identity, its license and the route it took to get there. Two things do need attention: -- **A component that ships no license text stops the build.** The fix is to obtain the - terms, not to record an assumption. +- **A component that states its license nowhere stops the build.** The fix is to obtain the + terms, not to record an assumption. Stating one only in its source headers is not that + case: the reference text for that identifier ships, labelled as a substitution. - **A new member that the build cannot explain stops the build.** That means something reached the wheel by a route the collector cannot see; teach it that route rather than excluding the file. diff --git a/scripts/sbom/sbom/licensing.py b/scripts/sbom/sbom/licensing.py index f7193cde15..d0b4bd5645 100644 --- a/scripts/sbom/sbom/licensing.py +++ b/scripts/sbom/sbom/licensing.py @@ -738,9 +738,14 @@ def _debian_copyright_fields(lines: list[str]) -> list[str]: """ found: list[str] = [] for index, line in enumerate(lines): - if line.rstrip() != "Copyright:": + if not line.startswith("Copyright:"): continue - for position, value in enumerate(lines[index + 1 :]): + # Debian writes the field either way, and the same-line form is the + # commoner by far. `Copyright: Jane Doe` matches no notice pattern -- + # the colon breaks it -- so without this such a package credits nobody. + inline = line[len("Copyright:") :].strip() + values = ([f" {inline}"] if inline else []) + list(lines[index + 1 :]) + for position, value in enumerate(values): if not value[:1].isspace() or not value.strip(): break holder = " ".join(value.split()).rstrip(".") diff --git a/scripts/sbom/sbom/report.py b/scripts/sbom/sbom/report.py index 95a75f69fd..5010d40d1a 100644 --- a/scripts/sbom/sbom/report.py +++ b/scripts/sbom/sbom/report.py @@ -112,12 +112,13 @@ def notices_markdown(inventory: Inventory, dist_info: str, wheel_name: str) -> s lines.append(f"- License: `{component.license_concluded}`") if component.copyright_text != "NOASSERTION": # A reference text names no holder, so for a component that ships no - # licence file of its own this is the only notice in the wheel. One - # per line, indented: a bare newline inside a list item is a lazy - # continuation and renders several notices as one paragraph. - notices = component.copyright_text.splitlines() - lines.append(f"- Copyright: {notices[0]}") - lines.extend(f" {item}" for item in notices[1:]) + # licence file of its own this is the only notice in the wheel. + # Joined with an explicit break: a newline inside a list item is a + # lazy continuation, and so is an indented one, so either way a + # renderer runs distinct legal notices into one paragraph. + lines.append( + "- Copyright: " + "
".join(component.copyright_text.splitlines()) + ) if component.license_declared != component.license_concluded: lines.append(f"- Declared: `{component.license_declared}`") if component.homepage != "NOASSERTION": @@ -296,7 +297,7 @@ def report_markdown(payload: dict) -> str: "None declared." ] - lines += ["", "## Fetched for this build but not in this wheel", ""] + lines += ["", "## Present in this build but not in this wheel", ""] if payload["exclusions"]: lines += [ f"- `{item['component']}` — {item['reason']}" diff --git a/tests/python/sbom/test_licensing.py b/tests/python/sbom/test_licensing.py index e411bdc967..87e9e5c88c 100644 --- a/tests/python/sbom/test_licensing.py +++ b/tests/python/sbom/test_licensing.py @@ -312,3 +312,62 @@ def test_a_template_is_not_a_notice(): licensing.read_notice("MIT\n\nCopyright (c) 2016 Wenzel Jakob\n\nPermission") == "Copyright (c) 2016 Wenzel Jakob" ) + + +def test_a_debian_copyright_field_names_its_holder_either_way(license_data): + """Debian states a holder in a field, in two spellings, often with no year. + + `Copyright: Jane Doe` matches no notice pattern -- the colon breaks it -- and + it is the commoner of the two forms, so a system library vendored from such a + package was credited to nobody. The field also holds prose and the literal + `None`, which are not holders. + """ + licensing.load_corpus(license_data) + + same_line = "Copyright: Poul-Henning Kamp \nLicense: Beerware\n" + indented = "Copyright:\n Colin Plumb\n Todd C. Miller\nLicense: public-domain\n" + + assert licensing.read_notices(same_line) == [ + "Poul-Henning Kamp " + ] + assert licensing.read_notices(indented) == ["Colin Plumb", "Todd C. Miller"] + assert licensing.read_notices("Copyright: None\nLicense: MIT\n") == [] + assert ( + licensing.read_notices( + "Copyright: This code is derived from software contributed by X\n" + ) + == [] + ) + + +def test_every_holder_in_a_text_is_read_not_just_the_first(license_data): + """Under MIT, BSD and Zlib the notice is the obligation, so all of it counts. + + Holders run on past a comma, sit on the next line with nothing joining them, + and follow a lead-in of any length. + """ + licensing.load_corpus(license_data) + + assert licensing.read_notices( + "Copyright (c) 2002-2006 Marcus Geelnard\n\n" + "Copyright (c) 2006-2019 Camilla Lowy\n" + ) == [ + "Copyright (c) 2002-2006 Marcus Geelnard", + "Copyright (c) 2006-2019 Camilla Lowy", + ] + # The holder on the line below, with no comma to join them. + assert licensing.read_notices( + "Copyright © 1980, 1982, 1986, 1989-1994\n" + " The Regents of the University of California.\n" + ) == [ + "Copyright © 1980, 1982, 1986, 1989-1994 The Regents of the University " + "of California." + ] + # A lead-in longer than a name, and a clause broken by the end of its line. + assert licensing.read_notices( + "Box collision code (engine_collision_box.c) is Copyright 2016 S Kolev.\n" + ) == ["Copyright 2016 S Kolev."] + assert licensing.read_notices( + "2010), this software is Copyright (c) 2007-2010 by B Lepilleur, and is\n" + "released under the terms of the MIT License.\n" + ) == ["Copyright (c) 2007-2010 by B Lepilleur"] From f32198f83181b9db5e9e326eb642907028b56a3e Mon Sep 17 00:00:00 2001 From: Andrei Aristarkhov Date: Thu, 1 Oct 2026 03:56:27 -0700 Subject: [PATCH 060/101] docs(agents): a rule changed in one place is changed in none A correction that lands in one of several places where the same fact is stated has failed, and this has been the repeated shape of it: a predicate enforced in four modules, a design doc contradicting its own collector, a per-row reason fixed under a heading that still said the opposite. Grep for the claim rather than the line; prefer one authority; check the output and not only the suite, which passed throughout. Signed-off-by: Andrei Aristarkhov --- AGENTS.md | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/AGENTS.md b/AGENTS.md index 06752ccd75..833c5fa0ed 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -266,6 +266,24 @@ first** if you are changing anything under `scripts/sbom/` or wheel-packaging steps in `build-ubuntu.yml`. It covers discovery, license matching and the maintenance rules; nothing here repeats them. +## A rule changed in one place is changed in none + +When a fact is stated in more than one place — a predicate in several modules, a +claim in both the code and the docs, a heading over the rows it describes — fixing +one instance and leaving the rest is the most common way a correction fails to +stick here. It has happened repeatedly: a licence rule enforced in four modules, a +design doc contradicting its own collector, a per-row reason corrected under a +heading that still said the opposite. + +- **Grep for the claim, not the line you edited.** Search the phrase, the + predicate, the constant — across `docs/`, `scripts/`, `tests/` and the code — + and fix every hit in the same pass. +- **Prefer one authority to a swept duplicate.** If the same rule has to hold in + several places, give it one definition and call it; the sweep is what you do + when you cannot. +- **Verify against output, not against the test suite.** Several of these passed + every test while producing wrong artifacts. Regenerate and look. + ## Mandatory learning loop (AGENTS.md and comments) **Hard requirement:** When **any** of the following happens, you **must** complete steps 1–3 **before** you end the session or move on as if the work were complete: From 1c018af631ed1a8dd37a0b1468f1d6bdba809138 Mon Sep 17 00:00:00 2001 From: Andrei Aristarkhov Date: Thu, 1 Oct 2026 04:08:59 -0700 Subject: [PATCH 061/101] build(sbom): stop a choice of licences being published as both `SPDX-License-Identifier: MIT OR GPL-2.0-or-later` came out of a component as `GPL-2.0-or-later AND MIT`, and out of the wheel the same way. A choice the author offered became both obligations at once, which for a copyleft option is the difference between a licence a consumer can take and one they cannot. The header's own expression is kept whole, and the roll-up decomposes an expression only where doing so cannot change it -- an `OR` stays a unit. A declared licence with an exception could never pass its gate: the pinned list carries exception texts separately and this build fetches only the licences, so `Apache-2.0 WITH LLVM-exception` demanded a text nothing could supply. An exception modifies the licence it is attached to, so it is not a text to ship. And the gate on the distribution's own terms matched on basename alone, so a packaged `third-party/openxr-sdk/LICENSES/Apache-2.0.txt` satisfied it. A wheel that stopped shipping its own LICENSE would have passed with its only copy of its terms attributed to OpenXR. Four smaller defects, each reproduced before the fix: a LicenseRef shared by three or more components named the last two and cited only the last one's files; a notice ending in a year took the first line of the licence body below it as the holder; two requirement lines naming one sibling emitted the same `externalDocumentId` twice, which SPDX forbids and nothing caught; and the dangling-clause strip was quadratic -- 128 KB of trailing connectives took 70 seconds, now 8 milliseconds, by trimming tokens rather than rematching. The CLI reports a corrupt manifest, an unparseable requirement, a `--wheel` that is not a zip and a missing file instead of printing a traceback. Signed-off-by: Andrei Aristarkhov --- scripts/sbom/sbom/build.py | 25 +++++++++-- scripts/sbom/sbom/cli.py | 10 +++++ scripts/sbom/sbom/discovery.py | 7 +++- scripts/sbom/sbom/document.py | 60 +++++++++++++------------- scripts/sbom/sbom/licensing.py | 65 ++++++++++++++++++++++++++--- tests/python/sbom/test_gates.py | 21 ++++++++++ tests/python/sbom/test_licensing.py | 44 +++++++++++++++++++ 7 files changed, 192 insertions(+), 40 deletions(-) diff --git a/scripts/sbom/sbom/build.py b/scripts/sbom/sbom/build.py index 9949663f01..7d1faf93c7 100644 --- a/scripts/sbom/sbom/build.py +++ b/scripts/sbom/sbom/build.py @@ -132,7 +132,25 @@ def _declared_without_text(declared: str, shipped: set[str]) -> list[str]: # An expression nothing can read names nothing that can be checked, and # the document carries it verbatim; say so rather than pass silently. return [declared] - return sorted(item for item in identifiers if f"{item.lower()}.txt" not in shipped) + return sorted( + item + for item in identifiers + # An exception modifies the licence it is attached to and has no text in + # the list this build fetches, so demanding one blocks every + # `X WITH ` declaration with an instruction nothing can meet. + if not licensing.is_exception(item) and f"{item.lower()}.txt" not in shipped + ) + + +def _own_texts(shipped: set[str]) -> set[str]: + """The texts a wheel carries as its own, not as a third party's. + + These are named relative to `.dist-info/licenses/`, so a third party's sits + under `third-party/`. Counting those let somebody else's copy discharge this + distribution's obligation -- remove that component and the wheel silently + loses its own terms. + """ + return {item for item in shipped if not item.startswith("third-party/")} def _own_license_texts(declared: str, shipped: set[str]) -> dict[str, str]: @@ -144,7 +162,7 @@ def _own_license_texts(declared: str, shipped: set[str]) -> dict[str, str]: """ if declared == NOASSERTION: return {} - names = {Path(item).name.lower() for item in shipped} + names = {Path(item).name.lower() for item in _own_texts(shipped)} wanted = {} for item in licensing.canonical_evidence("this-distribution", {declared}): identifier = item.identified @@ -358,8 +376,7 @@ def build( # pinned list does not carry -- is a declaration with no terms behind it. unmet = _declared_without_text( declared, - {Path(item).name.lower() for item in packaged_paths} - | {Path(item).name.lower() for item in shipped}, + {Path(item).name.lower() for item in _own_texts(set(packaged_paths) | shipped)}, ) if unmet: raise BuildError( diff --git a/scripts/sbom/sbom/cli.py b/scripts/sbom/sbom/cli.py index 9c4cd8de68..8ceb1ae04d 100644 --- a/scripts/sbom/sbom/cli.py +++ b/scripts/sbom/sbom/cli.py @@ -9,6 +9,7 @@ import json import subprocess import sys +import zipfile from pathlib import Path from packaging.utils import canonicalize_name @@ -221,6 +222,15 @@ def main(argv: list[str] | None = None) -> int: discovery.FileApiError, wheelfile.WheelError, validate_module.ManifestError, + # Malformed input reaches these through a document, a manifest or a + # wheel's own metadata: a corrupt JSON body, a requirement that will not + # parse, a `--wheel` that is not a zip, a manifest missing a key, a file + # that is not there. Each has something worth printing; a traceback + # buries it and exits 1 where the caller expects 2. + ValueError, + OSError, + KeyError, + zipfile.BadZipFile, ) as error: print(f"error: {error}", file=sys.stderr) return 2 diff --git a/scripts/sbom/sbom/discovery.py b/scripts/sbom/sbom/discovery.py index dd3014f2ea..b1a0a11d9b 100644 --- a/scripts/sbom/sbom/discovery.py +++ b/scripts/sbom/sbom/discovery.py @@ -604,7 +604,8 @@ class _Stated: def _stated_by(path: Path) -> _Stated | None: """Holder, licence and notice, from one read of the file's head.""" try: - head = path.open("rb").read(4096).decode("utf-8", "replace") + with path.open("rb") as handle: + head = handle.read(4096).decode("utf-8", "replace") except OSError: return None match = _COPYRIGHT_TAG.search(head) @@ -647,7 +648,11 @@ def discover_vendored( evidence = licensing.canonical_evidence(key, licences.get(key, set())) concluded, declared = licensing.expression(evidence, key) if licences.get(key): + # The headers' own expressions, structure and all. Rebuilding this + # from the reference texts the identifiers name turns `A OR B` into + # `A AND B` -- a choice restated as both obligations at once. declared = licensing.combine(sorted(licences[key])) + concluded = declared # Say when the list is shortened; without the marker it reads as the # whole set, and the first four by name need not be the ones that ship. named = sorted(str(item.relative_to(repo_root)) for item in paths) diff --git a/scripts/sbom/sbom/document.py b/scripts/sbom/sbom/document.py index 157a01878e..0425d327e4 100644 --- a/scripts/sbom/sbom/document.py +++ b/scripts/sbom/sbom/document.py @@ -117,26 +117,29 @@ def _extracted_entries(components: list[Component]) -> list[dict]: f"{ref} was minted for two components with different terms; " "a LicenseRef identifies one text" ) - shipped = ", ".join(item.path for item in grants) + # Accumulated, not replaced: three components sharing one text left + # the entry naming the last two and citing only the last one's files. + covers = set(entries.get(ref, {}).get("_for", ())) | {component.name} + cited = sorted( + set(entries.get(ref, {}).get("_from", ())) + | {item.path for item in grants} + ) entries[ref] = { "licenseId": ref, - "name": "License terms shipped with " - + ", ".join( - sorted( - {entries[ref]["_for"], component.name} - if ref in entries - else {component.name} - ) - ), - "_for": component.name, + "name": "License terms shipped with " + ", ".join(sorted(covers)), + "_for": covers, + "_from": cited, "extractedText": body, "comment": ( licensing.why_unnamed(grants) - + f"; reproduced here verbatim from {shipped}." + + "; reproduced here verbatim from " + + ", ".join(cited) + + "." ), } for entry in entries.values(): entry.pop("_for", None) + entry.pop("_from", None) return sorted(entries.values(), key=lambda item: item["licenseId"]) @@ -248,19 +251,17 @@ def _wheel_conclusion(declared: str, inventory: Inventory) -> str: said the wheel carries no code under them. Only a component concluding the declared licence itself adds nothing. """ - # By identifier, not by expression: combining whole expressions nests the - # compound ones in parentheses and repeats an id two components share. - identifiers: set[str] = set() + # Decomposed, so an id two components share appears once and a compound one + # does not arrive nested -- but never through an `OR`, which is one choice + # and becomes a claim on both halves if it is split. + operands: set[str] = set() for component in inventory.components_present.values(): if component.license_concluded == NOASSERTION: continue - try: - identifiers.update(licensing.identifiers_in(component.license_concluded)) - except licensing.ExpressionReadError: - identifiers.add(component.license_concluded) + operands.update(licensing.operands_of(component.license_concluded)) if declared != NOASSERTION: - identifiers.add(declared) - return licensing.combine(sorted(identifiers)) if identifiers else declared + operands.update(licensing.operands_of(declared)) + return licensing.combine(sorted(operands)) if operands else declared def build_document( @@ -499,16 +500,15 @@ def build_document( # local stub beside it would restate version, supplier and licence # from the same source. document_ref = f"DocumentRef-{licensing.spdx_safe(sibling['filename'])}" - external_documents.append( - { - "externalDocumentId": document_ref, - "spdxDocument": sibling["namespace"], - "checksum": { - "algorithm": "SHA1", - "checksumValue": sibling["sha1"], - }, - } - ) + # Two requirement lines can name one sibling -- a plain one and an + # extra-gated one -- and SPDX wants each externalDocumentId once. + entry = { + "externalDocumentId": document_ref, + "spdxDocument": sibling["namespace"], + "checksum": {"algorithm": "SHA1", "checksumValue": sibling["sha1"]}, + } + if entry not in external_documents: + external_documents.append(entry) element = f"{document_ref}:{WHEEL_PACKAGE_ID}" comment = ( f"{declared} Built by this same build as {sibling['name']} " diff --git a/scripts/sbom/sbom/licensing.py b/scripts/sbom/sbom/licensing.py index d0b4bd5645..5c6bf8cdcf 100644 --- a/scripts/sbom/sbom/licensing.py +++ b/scripts/sbom/sbom/licensing.py @@ -628,12 +628,16 @@ def fold_notices(notices) -> list[str]: re.MULTILINE | re.IGNORECASE, ) # A clause broken by the end of a line, left dangling by taking the line alone. -_DANGLING = re.compile(r"(?:[,;]?\s+(?:and|or|is|was|are|were|by|under|the|a))+$", re.I) +# Words a notice can end on only because the line broke mid-clause. +_DANGLING_WORDS = frozenset( + {"and", "or", "is", "was", "are", "were", "by", "under", "the", "a"} +) # Years, year ranges and the markers around them: what is left is the holder. # A sentence, as against a name: the Debian `Copyright:` field holds both. _PROSE = re.compile( r"\b(derived|contributed|reserved|portions?|redistribut\w+|written|" - r"modified|see|provided|this|these|some|all rights)\b", + r"modified|see|provided|permission|granted|warrant\w*|software|hereby|" + r"this|these|those|some|all rights)\b", re.IGNORECASE, ) _YEARS_AND_MARKS = re.compile(r"\(c\)|©|\d{4}(?:\s*-\s*\d{2,4})?|[,;]", re.I) @@ -714,12 +718,20 @@ def read_notices(text: str) -> list[str]: if ahead >= len(lines): break following = " ".join(lines[ahead].split()) - # A new notice, or a field of its own, is not this one's holder. - if not following or _PLAIN_COPYRIGHT.match(following) or ":" in following: + # A new notice, a field of its own, or a sentence is not a holder: + # a year-only notice followed by the licence body would otherwise + # take its first line as the name. + if ( + not following + or _PLAIN_COPYRIGHT.match(following) + or ":" in following + or _PROSE.search(following) + or _NOT_A_NOTICE.search(following) + ): break step = ahead notice = f"{notice} {following}".strip().rstrip("*/ ") - notice = _DANGLING.sub("", notice).rstrip("*/,; ") + notice = _without_dangling(notice) if _PLACEHOLDER.search(notice) or _NOT_A_NOTICE.search(notice): continue if not _names_a_holder(notice) and not re.search(r"\d{4}", notice): @@ -764,6 +776,18 @@ def _debian_copyright_fields(lines: list[str]) -> list[str]: return found +def _without_dangling(notice: str) -> str: + """Drop the connectives a notice ends on because its line broke mid-clause. + + By token, not by regex: a repeated or nested match over the whole string is + quadratic on a long line that ends in many of them. + """ + words = notice.split() + while words and words[-1].rstrip(",;").lower() in _DANGLING_WORDS: + words.pop() + return " ".join(words).rstrip("*/,; ") + + def _names_a_holder(notice: str) -> bool: """Whether anything is left once the years and their punctuation come out.""" rest = _YEARS_AND_MARKS.sub(" ", notice[len("Copyright") :]) @@ -873,6 +897,37 @@ def rename_refs(expression: str, renamed: dict[str, str]) -> str: return expression +def is_exception(identifier: str) -> bool: + """Whether an identifier is a licence exception rather than a licence. + + An exception modifies the licence it is attached to; the pinned list ships + their texts separately and this build fetches only the licences. + """ + try: + parsed = _spdx_licensing().parse(identifier, validate=False) + except (ExpressionError, ValueError, TypeError): + return False + return identifier in {str(item) for item in getattr(parsed, "symbols", ())} and any( + getattr(item, "is_exception", False) for item in parsed.symbols + ) + + +def operands_of(expression: str) -> list[str]: + """An expression split into parts that may be AND-joined without changing it. + + An `OR` is one choice and splitting it asserts both halves, so it stays + whole. Everything else decomposes to identifiers, which keeps a roll-up flat + and free of an id two components happen to share. + """ + try: + parsed = _spdx_licensing().parse(expression, validate=False) + except (ExpressionError, ValueError, TypeError): + return [expression] + if " OR " in str(parsed).upper(): + return [f"({parsed})"] + return sorted(str(key) for key in _spdx_licensing().license_symbols(parsed)) + + def identifiers_in(expression: str) -> list[str]: """Every licence identifier an expression names, LicenseRef included.""" try: diff --git a/tests/python/sbom/test_gates.py b/tests/python/sbom/test_gates.py index d7c9d457d8..d20626f902 100644 --- a/tests/python/sbom/test_gates.py +++ b/tests/python/sbom/test_gates.py @@ -505,3 +505,24 @@ def components(wheel_name): "a wheel that ships no such library must not inherit it" ) assert validate_module.check(workspace.wheel) == [] + + +def test_a_third_partys_licence_does_not_discharge_this_one(workspace, license_data): + """The distribution's own terms must be its own. + + A packaged third-party text with a matching basename satisfied the gate, so + a wheel that stopped shipping its own LICENSE still passed and the only copy + of its terms sat under `third-party/`, attributed to somebody else. + """ + from sbom import build as build_module + + licensing.load_corpus(license_data) + assert build_module._own_license_texts( + "Apache-2.0", {"third-party/openxr-sdk/LICENSES/Apache-2.0.txt"} + ), "a third party's copy must not count as this distribution's" + assert not build_module._own_license_texts( + "Apache-2.0", {"LICENSES/Apache-2.0.txt"} + ), "the distribution's own copy counts" + assert build_module._declared_without_text( + "Apache-2.0", {"third-party/openxr-sdk/licenses/apache-2.0.txt"} + ) == ["Apache-2.0"] diff --git a/tests/python/sbom/test_licensing.py b/tests/python/sbom/test_licensing.py index 87e9e5c88c..52cbde7d20 100644 --- a/tests/python/sbom/test_licensing.py +++ b/tests/python/sbom/test_licensing.py @@ -371,3 +371,47 @@ def test_every_holder_in_a_text_is_read_not_just_the_first(license_data): "2010), this software is Copyright (c) 2007-2010 by B Lepilleur, and is\n" "released under the terms of the MIT License.\n" ) == ["Copyright (c) 2007-2010 by B Lepilleur"] + + +def test_a_choice_of_licences_is_not_published_as_both(license_data, tmp_path): + """`A OR B` is one licence at the reader's choice, not two obligations. + + Rebuilding a component's conclusion from the reference texts its identifiers + name loses the operator, and the roll-up that splits an expression into + identifiers loses it again. + """ + from sbom import discovery + + licensing.load_corpus(license_data) + source = tmp_path / "vendored.h" + source.write_text( + "// SPDX-FileCopyrightText: Copyright 2020 Upstream Widgets Ltd\n" + "// SPDX-License-Identifier: MIT OR GPL-2.0-or-later\n" + ) + components, _ = discovery.discover_vendored(tmp_path, {source}, {"NVIDIA"}) + component = next(iter(components.values())) + + assert component.license_concluded == "MIT OR GPL-2.0-or-later" + assert component.license_declared == "MIT OR GPL-2.0-or-later" + # And it survives the roll-up whole, where an AND-only one decomposes. + assert licensing.operands_of("MIT OR GPL-2.0-or-later") == [ + "(MIT OR GPL-2.0-or-later)" + ] + assert licensing.operands_of("Apache-2.0 AND CC-BY-4.0") == [ + "Apache-2.0", + "CC-BY-4.0", + ] + + +def test_an_exception_is_not_a_licence_needing_its_own_text(license_data): + """`X WITH ` must not be unsatisfiable. + + The pinned list ships exception texts separately and this build fetches only + the licences, so demanding a text for the exception blocks such a + declaration with an instruction nothing in the collector can meet. + """ + licensing.load_corpus(license_data) + + assert licensing.is_exception("LLVM-exception") + assert licensing.is_exception("Classpath-exception-2.0") + assert not licensing.is_exception("Apache-2.0") From 0cc299d9ec58f2555944bf70d3e62db18f8421aa Mon Sep 17 00:00:00 2001 From: Andrei Aristarkhov Date: Thu, 1 Oct 2026 09:13:23 -0700 Subject: [PATCH 062/101] build(sbom): name the licensor of a wheel that ships no file of its own The transition wheel's own package stated `copyrightText: NOASSERTION`, and its notices file said nothing about the distribution at all. The only text in that wheel is the Apache-2.0 reference the collector packages for the licence it declares, whose appendix reads `Copyright [yyyy] [name of copyright owner]` -- so a recipient holding the wheel could not name who licensed it to them, for a wheel that declares a licence. The roll-up reads notices from the files a wheel ships, and a metadata-only wheel ships none. What the project states about itself is observable all the same: the REUSE header on the packaging metadata the distribution was built from. That is the fallback now, and the notices file opens with the licence and the holder rather than going straight to third parties -- it is the artifact a person reads, and it was the one saying least about the wheel it travels in. Signed-off-by: Andrei Aristarkhov --- docs/source/references/sbom.rst | 6 ++++ scripts/sbom/sbom/build.py | 10 ++++-- scripts/sbom/sbom/discovery.py | 19 ++++++++++ scripts/sbom/sbom/document.py | 10 +++--- scripts/sbom/sbom/report.py | 31 +++++++++++++--- tests/python/sbom/synth.py | 3 ++ tests/python/sbom/test_wheel_sbom.py | 53 ++++++++++++++++++++++++++++ 7 files changed, 122 insertions(+), 10 deletions(-) diff --git a/docs/source/references/sbom.rst b/docs/source/references/sbom.rst index 378d11d438..21fca9d594 100644 --- a/docs/source/references/sbom.rst +++ b/docs/source/references/sbom.rst @@ -218,6 +218,12 @@ The distribution's own declared licence is an obligation like any other. A wheel redistributed component with no terms; where the wheel ships none, the reference text for each identifier is packaged from the pinned list and declared under PEP 639. +The notices file opens with the distribution's own licence and copyright holder, read +from the packaging metadata it was built from where no file it ships states one. A wheel +that ships only metadata carries no notice otherwise: the reference text its declared +licence packages is a template naming nobody, so a recipient holding the wheel alone could +not say who licensed it to them. + ``licenseDeclared`` on the wheel is what the distribution says of itself. ``licenseConcluded`` is what this document found in it, so a wheel carrying a component whose terms the build could not name states that expression alongside the declaration — a reader asking "what is diff --git a/scripts/sbom/sbom/build.py b/scripts/sbom/sbom/build.py index 7d1faf93c7..3886ae8184 100644 --- a/scripts/sbom/sbom/build.py +++ b/scripts/sbom/sbom/build.py @@ -25,7 +25,7 @@ from packaging.utils import canonicalize_name from . import discovery as discovery_module -from .discovery import project_supplier, sha256_file +from .discovery import project_copyright, project_supplier, sha256_file from .document import NOASSERTION from .inventory import Attribution from .wheelfile import Entry @@ -61,6 +61,9 @@ def _project_facts(repo_root: Path, metadata: dict) -> dict: # expression, and a wheel declaring neither declares nothing. "license": metadata.get("license_expression") or NOASSERTION, "homepage": homepage, + # What the project states about itself, for a distribution whose own + # files state nothing -- a metadata-only wheel has no file to carry it. + "copyright": project_copyright(repo_root), } @@ -366,7 +369,10 @@ def build( notices_path = report_module.notices_path(wheel.dist_info) additions[notices_path] = report_module.notices_markdown( - inventory, wheel.dist_info, wheel_path.name + inventory, + wheel.dist_info, + wheel_path.name, + _project_facts(repo_root, wheel_metadata), ).encode("utf-8") packaged_paths.append( report_module.declared_license_path(wheel.dist_info, notices_path) diff --git a/scripts/sbom/sbom/discovery.py b/scripts/sbom/sbom/discovery.py index b1a0a11d9b..3d4193f240 100644 --- a/scripts/sbom/sbom/discovery.py +++ b/scripts/sbom/sbom/discovery.py @@ -592,6 +592,25 @@ def project_supplier(repo_root: Path) -> str: return f"Organization: {authors[0]}" +def project_copyright(repo_root: Path) -> str: + """The notice this project states about itself, from its packaging metadata. + + A distribution that ships files of its own is covered by their notices. One + that ships only metadata states its copyright nowhere a reader can reach -- + the reference text its declared licence packages names no holder -- and the + REUSE header on the file the distribution is built from is where the project + does state it. + """ + pyproject = repo_root / "pyproject.toml" + if not pyproject.is_file(): + return "NOASSERTION" + try: + head = pyproject.read_text(encoding="utf-8", errors="replace")[:4096] + except OSError: + return "NOASSERTION" + return licensing.read_copyright(head) or "NOASSERTION" + + @dataclass(frozen=True) class _Stated: """What a source file says about itself in its header.""" diff --git a/scripts/sbom/sbom/document.py b/scripts/sbom/sbom/document.py index 0425d327e4..a0ce1e1c34 100644 --- a/scripts/sbom/sbom/document.py +++ b/scripts/sbom/sbom/document.py @@ -359,11 +359,13 @@ def build_document( "licenseConcluded": _wheel_conclusion(project["license"], inventory), "licenseDeclared": project["license"], "licenseInfoFromFiles": sorted(stated_in_files) or [NOASSERTION], - # The notices its own files carry. Declaring none beside a supplier, a - # licence and files that each state one left the package saying less - # about itself than anything in it. + # The notices its own files carry, or -- for a distribution that ships + # no file of its own -- what the project states in the metadata it is + # built from. Declaring none beside a supplier and a licence left the + # package saying less about itself than anything in it, and left a + # recipient of a metadata-only wheel unable to name its licensor at all. "copyrightText": "\n".join(licensing.fold_notices(stated_copyright)) - or NOASSERTION, + or project.get("copyright", NOASSERTION), "packageVerificationCode": { "packageVerificationCodeValue": code, "packageVerificationCodeExcludedFiles": [ diff --git a/scripts/sbom/sbom/report.py b/scripts/sbom/sbom/report.py index 5010d40d1a..7606cd31ce 100644 --- a/scripts/sbom/sbom/report.py +++ b/scripts/sbom/sbom/report.py @@ -10,6 +10,8 @@ from . import licensing from .inventory import Inventory +NOASSERTION = "NOASSERTION" + _HOW_PROSE = { "compiled-source": "its sources were compiled into a shipped binary", "header-include": "its headers were compiled into a shipped binary", @@ -87,9 +89,30 @@ def _roles(inventory: Inventory, key: str) -> str: return "; ".join(_HOW_PROSE.get(how, how) for how in hows) or "unrecorded" -def notices_markdown(inventory: Inventory, dist_info: str, wheel_name: str) -> str: - """The notice file that travels inside the wheel.""" - lines = ["# Third-party notices", ""] +def notices_markdown( + inventory: Inventory, + dist_info: str, + wheel_name: str, + project: dict | None = None, +) -> str: + """The notice file that travels inside the wheel. + + It opens with the distribution's own licence and holder. The reference text + a declared licence packages names nobody -- Apache-2.0's appendix is a + template -- so for a wheel shipping no source file of its own this is the + only place a recipient can read who licensed it to them. + """ + lines = ["# Notices", ""] + if project: + # `Organization: NVIDIA` is how SPDX spells a supplier; prose does not. + supplier = project["supplier"].split(": ", 1)[-1] + lines += [f"`{wheel_name}` is distributed by {supplier}."] + if project.get("license", NOASSERTION) != NOASSERTION: + lines.append(f"It is licensed under `{project['license']}`.") + if project.get("copyright", NOASSERTION) != NOASSERTION: + lines.append(f"{project['copyright']}") + lines.append("") + lines += ["## Third-party notices", ""] if not inventory.components_present: # Saying where texts live when none were written sends a reader to a # directory the wheel does not contain. @@ -107,7 +130,7 @@ def notices_markdown(inventory: Inventory, dist_info: str, wheel_name: str) -> s "", ] for key, component in inventory.components_present.items(): - lines.append(f"## {component.name}") + lines.append(f"### {component.name}") lines.append("") lines.append(f"- License: `{component.license_concluded}`") if component.copyright_text != "NOASSERTION": diff --git a/tests/python/sbom/synth.py b/tests/python/sbom/synth.py index 3884bcfbda..d53390ddec 100644 --- a/tests/python/sbom/synth.py +++ b/tests/python/sbom/synth.py @@ -251,6 +251,9 @@ def retarget(): """ _PROJECT_PYPROJECT = """\ +# SPDX-FileCopyrightText: Copyright (c) 2026 Example Org. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + [project] name = "isaaccapture" version = "0.4.0" diff --git a/tests/python/sbom/test_wheel_sbom.py b/tests/python/sbom/test_wheel_sbom.py index 5309a141c8..a48f7d043a 100644 --- a/tests/python/sbom/test_wheel_sbom.py +++ b/tests/python/sbom/test_wheel_sbom.py @@ -468,3 +468,56 @@ def rewrite(spdx): built["wheel"], build_evidence=built["evidence_path"] ) assert any("not the terms the build read" in item for item in failures), failures + + +def test_a_wheel_shipping_no_source_still_names_its_licensor(workspace, license_data): + """The declared licence's reference text names nobody. + + A metadata-only wheel ships no file carrying a notice, so without this the + SPDX package states NOASSERTION and the notices file says nothing about the + distribution -- a recipient holding only the wheel cannot tell who licensed + it to them, for a wheel that declares one. + """ + thin = workspace.wheel.parent / "thin-0.4.0-py3-none-any.whl" + synth.write_wheel( + thin, + { + "thin-0.4.0.dist-info/METADATA": "\n".join( + [ + "Metadata-Version: 2.4", + "Name: thin", + "Version: 0.4.0", + "Author: Example Org", + "License-Expression: Apache-2.0", + "", + ] + ).encode(), + "thin-0.4.0.dist-info/WHEEL": ( + b"Wheel-Version: 1.0\nGenerator: setuptools\nRoot-Is-Purelib: true\n" + ), + }, + ) + out = workspace.root / "sbom" + build_module.build(workspace.root, workspace.build, thin, out, license_data) + + with zipfile.ZipFile(thin) as archive: + spdx = json.loads( + archive.read( + "thin-0.4.0.dist-info/sboms/thin-0.4.0-py3-none-any.whl.spdx.json" + ) + ) + notices = archive.read( + "thin-0.4.0.dist-info/licenses/THIRD-PARTY-NOTICES.md" + ).decode() + members = set(archive.namelist()) + + wheel_package = next( + item for item in spdx["packages"] if item["SPDXID"] == "SPDXRef-Package-wheel" + ) + assert wheel_package["copyrightText"] != "NOASSERTION" + assert "Copyright" in notices + assert "Example Org" in notices + # The SPDX spelling of a supplier is not prose. + assert "distributed by Organization:" not in notices + # The only text it ships names no holder, which is why the above matters. + assert "thin-0.4.0.dist-info/licenses/LICENSES/Apache-2.0.txt" in members From b9166477d7af985a98a0922a272149db4f19a70d Mon Sep 17 00:00:00 2001 From: Andrei Aristarkhov Date: Thu, 1 Oct 2026 09:22:37 -0700 Subject: [PATCH 063/101] build(sbom): say that most of this wheel's files determined no licence `licenseInfoFromFiles` listed the two identifiers REUSE tags stated and nothing else, while 99 of 240 files concluded NOASSERTION -- most of the wheel is binaries, which state nothing about themselves. Listing only what was found reads as every file's licence having been determined. NOASSERTION is what the rest of them found, so it is in the list. Signed-off-by: Andrei Aristarkhov --- scripts/sbom/sbom/document.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/scripts/sbom/sbom/document.py b/scripts/sbom/sbom/document.py index a0ce1e1c34..6f9bc0601e 100644 --- a/scripts/sbom/sbom/document.py +++ b/scripts/sbom/sbom/document.py @@ -315,6 +315,11 @@ def build_document( record["licenseConcluded"] = attribution.spdx_tag record["licenseInfoInFiles"] = [attribution.spdx_tag] stated_in_files.add(attribution.spdx_tag) + else: + # A binary states nothing about itself, and most of this wheel is + # binaries. Listing only what was found says every file's licence + # was determined; NOASSERTION is what the rest of them found. + stated_in_files.add(NOASSERTION) if ( attribution and attribution.copyright_text From 9489f0de173d083fbcaacf1ac7db2388bd170658 Mon Sep 17 00:00:00 2001 From: Andrei Aristarkhov Date: Thu, 1 Oct 2026 09:27:57 -0700 Subject: [PATCH 064/101] chore: stop ignoring the SBOM audit notes The file stays untracked, but by choice at the point of staging rather than by a rule. An ignore entry names it in the published diff anyway, which is most of what keeping it out was for. Signed-off-by: Andrei Aristarkhov --- .gitignore | 3 --- 1 file changed, 3 deletions(-) diff --git a/.gitignore b/.gitignore index 0bc18af015..c7ce6a078e 100644 --- a/.gitignore +++ b/.gitignore @@ -90,6 +90,3 @@ tmp/ !/examples/mujoco_xr/python/isaacteleop_examples/mujoco_xr/assets/follower/follower_arm.xml # Wheel SBOM and license evidence written by the collector in scripts/sbom/ /sbom/ - -# Audit notes kept with the branch, deliberately not published. -docs/sbom-open-findings.md From 0a944ad69d7df77306af06fc5c60cd9f44998c05 Mon Sep 17 00:00:00 2001 From: Andrei Aristarkhov Date: Thu, 1 Oct 2026 09:58:00 -0700 Subject: [PATCH 065/101] build: ask for the codemodel without raising the project's CMake minimum Raising `cmake_minimum_required` from `3.24...3.25` to `3.27` opts into every policy up to 3.27, and CMP0148 is one of them: it removes the FindPythonInterp module that pybind11's legacy path still calls. Under vcpkg's `find_package` wrapper that call then resolves in config-only mode and configure fails with "Could not find a package configuration file provided by PythonInterp". It configures without vcpkg, which is why a local build did not show it. `cmake_file_api()` is the only thing here that wanted 3.27, so it is feature-tested instead. Policies stay where main has them, and a CMake too old for the command writes no codemodel -- which the collector already reports. Signed-off-by: Andrei Aristarkhov --- CMakeLists.txt | 22 ++++++++++++---------- docs/source/references/sbom.rst | 6 +++--- scripts/sbom/sbom/discovery.py | 7 ++++--- 3 files changed, 19 insertions(+), 16 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 61fe9b98d5..85ff210923 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,16 +1,18 @@ # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -# 3.27 for cmake_file_api(): the SBOM collector reads this build's link and -# compile graph from the file API reply, and that command is what lets the -# query live here rather than in whatever invoked cmake. No range: raising the -# minimum already opts into every policy through 3.27, and nothing here needs -# the ones after it. -cmake_minimum_required(VERSION 3.27) - -# Ask for the codemodel this configure run. Generator-independent, and produced -# by the same run that writes it -- see docs: references/sbom. -cmake_file_api(QUERY API_VERSION 1 CODEMODEL 2) +cmake_minimum_required(VERSION 3.24...3.25) + +# The SBOM collector reads this build's link and compile graph from the file API +# reply, and `cmake_file_api()` is what lets the query live here rather than in +# whatever invoked cmake. It arrived in 3.27, and is feature-tested rather than +# required: raising the project minimum opts into every policy up to it, and +# CMP0148 -- which removes the FindPythonInterp module pybind11's legacy path +# still calls through vcpkg's find_package wrapper -- is one of them. Where the +# command is absent no codemodel is written, and the collector says so itself. +if(COMMAND cmake_file_api) + cmake_file_api(QUERY API_VERSION 1 CODEMODEL 2) +endif() # Workaround for older CMake policy compatibility in dependencies set(CMAKE_POLICY_VERSION_MINIMUM 3.5 CACHE STRING "Minimum CMake policy version") diff --git a/docs/source/references/sbom.rst b/docs/source/references/sbom.rst index 21fca9d594..afaa69a08a 100644 --- a/docs/source/references/sbom.rst +++ b/docs/source/references/sbom.rst @@ -150,9 +150,9 @@ CMake's file API codemodel binary, which targets it depends on, and — through include paths — which header-only dependencies it was compiled against. Followed transitively, so a library linked into a library linked into the wheel is still reported. The top-level ``CMakeLists.txt`` asks - for it with ``cmake_file_api()``, which is why the project requires CMake 3.27; reading - the generator's own output instead would only work under Makefiles, since Ninja writes - no ``link.txt``. + for it with ``cmake_file_api()``, which CMake 3.27 added — so a wheel with an SBOM + needs at least that, though the project builds without it. Reading the generator's own + output instead would only work under Makefiles, since Ninja writes no ``link.txt``. Archives in the repository Every member of every tarball is hashed. A wheel member that matches one is recorded as diff --git a/scripts/sbom/sbom/discovery.py b/scripts/sbom/sbom/discovery.py index 3d4193f240..196da9bf74 100644 --- a/scripts/sbom/sbom/discovery.py +++ b/scripts/sbom/sbom/discovery.py @@ -324,9 +324,10 @@ class BuildGraph: """The link and compile graph, read from CMake's own file API. The top-level ``CMakeLists.txt`` asks for the codemodel with - ``cmake_file_api()``, so any configure of this project leaves one behind, - under every generator. Scraping ``link.txt`` instead would only work for the - Makefile generators -- Ninja writes no such file. + ``cmake_file_api()`` where that command exists, so a configure by CMake 3.27 + or newer leaves one behind, under every generator. Scraping ``link.txt`` + instead would only work for the Makefile generators -- Ninja writes no such + file. """ def __init__(self, build_dir: Path, config: str | None = None) -> None: From fc02746786ea7bd3995dc5a4f609d29a2cc4e537 Mon Sep 17 00:00:00 2001 From: Andrei Aristarkhov Date: Thu, 1 Oct 2026 10:14:12 -0700 Subject: [PATCH 066/101] ci: keep a fetched archive's provenance in the cache with the archive This branch made the download script record the URL each CloudXR tarball came from, in a `.source` file beside it, and the collector refuses to let an archive explain a wheel member unless the repository tracks it or that marker says where it was fetched from. The cache stores only the tarballs, so a cache hit restored them without their provenance and every CloudXR member in the wheel became unattributable -- which is what the arm64 job just failed on. The marker is part of the fetched artifact, so it is cached with it. The key gains a version: the entries already saved hold tarballs and no markers, and a hit on one of those would fail exactly the same way. Signed-off-by: Andrei Aristarkhov --- .github/actions/setup-cloudxr-sdk/action.yml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/actions/setup-cloudxr-sdk/action.yml b/.github/actions/setup-cloudxr-sdk/action.yml index 0532ec0518..1c50ea02df 100644 --- a/.github/actions/setup-cloudxr-sdk/action.yml +++ b/.github/actions/setup-cloudxr-sdk/action.yml @@ -42,7 +42,12 @@ runs: path: | deps/cloudxr/*.tar.gz deps/cloudxr/*.tgz - key: cloudxr-sdk-${{ runner.os }}-${{ runner.arch }}-rt${{ steps.parse.outputs.runtime_version }}-web${{ steps.parse.outputs.web_version }} + # The URL each tarball came from, written beside it on download. A + # cache hit without it leaves every CloudXR wheel member unattributable: + # an archive explains one only if the repository tracks it or this + # marker says where it was fetched from. + deps/cloudxr/*.source + key: cloudxr-sdk-v2-${{ runner.os }}-${{ runner.arch }}-rt${{ steps.parse.outputs.runtime_version }}-web${{ steps.parse.outputs.web_version }} - name: Download CloudXR Runtime SDK shell: bash @@ -66,4 +71,5 @@ runs: path: | deps/cloudxr/*.tar.gz deps/cloudxr/*.tgz + deps/cloudxr/*.source key: ${{ steps.cache.outputs.cache-primary-key }} From 7eb4bc9c5b6820901d531ea140a3d4f33096c7e3 Mon Sep 17 00:00:00 2001 From: Andrei Aristarkhov Date: Thu, 1 Oct 2026 10:31:35 -0700 Subject: [PATCH 067/101] build(sbom): do not read the build tree as repository source vcpkg installs its ports under the build directory, and the codemodel cites their headers as compiled sources. The vendored scan admitted anything under the checkout that was not in `_deps`, so those headers were read as third-party code this repository had vendored -- one component per copyright line in them. The arm64 job produced about seventy, from Eigen, zlib, zstd, libarchive, fmt and spdlog contributors, and then failed the gate because no such "component" ships a licence text of its own. Include directories were already excluded from the build tree; sources were not. Both go through one predicate now, which also resolves its roots -- `--build-dir` is free-form, and a relative one compared against absolute paths excludes nothing. The seven components the checkout really does vendor are unaffected. Signed-off-by: Andrei Aristarkhov --- scripts/sbom/sbom/discovery.py | 48 +++++++++++++++++++---------- tests/python/sbom/synth.py | 17 +++++++++- tests/python/sbom/test_discovery.py | 22 +++++++++++++ 3 files changed, 70 insertions(+), 17 deletions(-) diff --git a/scripts/sbom/sbom/discovery.py b/scripts/sbom/sbom/discovery.py index 196da9bf74..02a4bc17e4 100644 --- a/scripts/sbom/sbom/discovery.py +++ b/scripts/sbom/sbom/discovery.py @@ -320,6 +320,22 @@ class FileApiError(Exception): """The build tree carries no CMake file API reply to read.""" +def _within(repo_root: Path, build_dir: Path): + """Predicate: under the checkout, outside its build tree. + + Both roots are resolved by the caller; `--build-dir` is free-form and a + relative one compared against absolute paths matches nothing. + """ + + def check(path: Path) -> bool: + resolved = path if path.is_absolute() else path.resolve() + return resolved.is_relative_to(repo_root) and not resolved.is_relative_to( + build_dir + ) + + return check + + class BuildGraph: """The link and compile graph, read from CMake's own file API. @@ -392,26 +408,26 @@ def by_name(self, name: str) -> list[Artifact]: return self._by_name.get(name, []) def repo_inputs(self, repo_root: Path, build_dir: Path) -> set[Path]: - """Files under `repo_root` this build compiled or included. - - `_deps` is the component-source domain and is attributed from its own - checkouts; an include directory under the build tree holds generated - headers, which state no upstream holder. + """Files this build compiled or included that the checkout itself holds. + + The build tree is not among them, for sources as much as for include + directories: `_deps` is the component-source domain and is attributed + from its own checkouts, generated headers state no upstream holder, and + vcpkg installs its ports under the build directory -- whose headers + carry their upstream contributors' copyright lines, which the vendored + scan reads as this repository having vendored each of those people. """ + inside = _within(repo_root.resolve(), build_dir.resolve()) found: set[Path] = set() for node in self._nodes.values(): - found.update( - source - for source in node.sources - if str(source).startswith(str(repo_root)) - and "/_deps/" not in source.as_posix() - ) + found.update(item for item in node.sources if inside(item)) for include in node.includes: - text = include.as_posix() - if not text.startswith(str(repo_root)) or "/_deps/" in text: - continue - if include.is_dir() and not text.startswith(str(build_dir)): - found.update(item for item in include.rglob("*") if item.is_file()) + if inside(include) and include.is_dir(): + found.update( + item + for item in include.rglob("*") + if item.is_file() and inside(item) + ) return found def _closure(self, target_id: str) -> tuple[_Node, ...]: diff --git a/tests/python/sbom/synth.py b/tests/python/sbom/synth.py index d53390ddec..158d205762 100644 --- a/tests/python/sbom/synth.py +++ b/tests/python/sbom/synth.py @@ -414,6 +414,13 @@ def create(root: Path) -> Workspace: _write(build / "generated/stubs/isaaccapture/py.typed", b"") # Staged tree: what the wheel is built from. + # What a package manager installs into the build tree: upstream headers + # carrying their own contributors' copyright lines. + _write( + build / "vcpkg_installed/x64-linux/include/eigen/Core.h", + b"// Copyright (C) 2008 Gael Guennebaud \n", + ) + _write(workspace.staged / "isaaccapture/__init__.py", _AUTHORED_INIT) _write(workspace.staged / "isaaccapture/_generated.py", _GENERATED) _write(workspace.staged / "isaaccapture/py.typed", b"") @@ -452,10 +459,18 @@ def _file_api_reply(root: Path, build: Path, extension: str) -> None: # The project's own source, compiled against beta's headers, with # alpha's source compiled straight in. "artifacts": [extension], - "sources": ["src/cpp/module.cpp", "build/_deps/alpha-src/alpha.cpp"], + # The vcpkg port is listed as a source, not just an include: a + # package manager installs into the build tree, and the codemodel + # cites what it compiled. + "sources": [ + "src/cpp/module.cpp", + "build/_deps/alpha-src/alpha.cpp", + "build/vcpkg_installed/x64-linux/include/eigen/Core.h", + ], "includes": [ "build/_deps/beta-src/include", "build/_deps/alpha-src", + "build/vcpkg_installed/x64-linux/include", "src/vendor", ], "dependencies": [], diff --git a/tests/python/sbom/test_discovery.py b/tests/python/sbom/test_discovery.py index 40e67a634e..721b44df0c 100644 --- a/tests/python/sbom/test_discovery.py +++ b/tests/python/sbom/test_discovery.py @@ -328,3 +328,25 @@ def test_a_relative_build_dir_still_keeps_the_staged_tree_out( assert indexed, "the build tree is indexed" assert not any("/python_package/" in item for item in indexed) + + +def test_a_dependency_installed_into_the_build_tree_is_not_vendored_source( + workspace, license_data +): + """vcpkg installs its ports under the build directory. + + Those headers carry their upstream contributors' copyright lines. Read as + repository source they become one "vendored" component per person -- seventy + of them on a real build -- each then failing the gate for shipping no licence + text of its own. + """ + from sbom import evidence, licensing + + licensing.load_corpus(license_data) + port = workspace.build / "vcpkg_installed/x64-linux/include/eigen/Core.h" + assert port.is_file(), "the fixture no longer installs a port into the build tree" + + discovered = evidence.discover(workspace.root, workspace.build) + holders = [key for key in discovered.components if key.startswith("vendored:")] + + assert not any("guennebaud" in key for key in holders), holders From 40909bb4cf7f66f85196eee61a6c1a4135414b31 Mon Sep 17 00:00:00 2001 From: Andrei Aristarkhov Date: Thu, 1 Oct 2026 11:22:09 -0700 Subject: [PATCH 068/101] build(sbom): stop naming a package after a copyright holder Checked-in third-party material is grouped by the copyright its files state, because that is the only identity such material has. From that holder the document also minted `pkg:generic/collabora-ltd`, `pkg:generic/wuji-technology` and the like -- package identifiers for packages that do not exist, which a reader resolves to nothing or to something else entirely. The contrast is in the same document: `pkg:github/khronosgroup/openxr-sdk` and `pkg:pypi/wuji-sdk` are real, and are not these. A component with nothing to locate now carries no `externalRefs` at all. The supplier still names the holder, which was observed. The licence report says what each row is -- fetched source, fetched archive, build machine, or vendored source named by its copyright holder -- so an organisation in the Component column is not read as a package. Signed-off-by: Andrei Aristarkhov --- scripts/sbom/sbom/discovery.py | 8 +++++--- scripts/sbom/sbom/document.py | 18 +++++++++++++----- scripts/sbom/sbom/report.py | 17 ++++++++++++++--- tests/python/sbom/test_discovery.py | 28 ++++++++++++++++++++++++++++ 4 files changed, 60 insertions(+), 11 deletions(-) diff --git a/scripts/sbom/sbom/discovery.py b/scripts/sbom/sbom/discovery.py index 02a4bc17e4..e76c1a097c 100644 --- a/scripts/sbom/sbom/discovery.py +++ b/scripts/sbom/sbom/discovery.py @@ -701,9 +701,11 @@ def discover_vendored( kind="vendored-source", supplier=f"Organization: {names[key]}", homepage="NOASSERTION", - purl=PackageURL( - type="generic", name=licensing.spdx_safe(names[key]).lower() - ).to_string(), + # No purl: this is third-party material identified by the copyright + # its files state, and there is no package to locate. Minting + # `pkg:generic/` named a package that does not exist, from + # an organisation rather than from anything shipped. + purl="NOASSERTION", version="NOASSERTION", download_location="NOASSERTION", source_info=( diff --git a/scripts/sbom/sbom/document.py b/scripts/sbom/sbom/document.py index 6f9bc0601e..0308e779ee 100644 --- a/scripts/sbom/sbom/document.py +++ b/scripts/sbom/sbom/document.py @@ -85,13 +85,21 @@ def _component_package(component: Component, roles: set[str], dist_info: str) -> "licenseConcluded": component.license_concluded, "licenseDeclared": component.license_declared, "copyrightText": component.copyright_text, - "externalRefs": [ + # Only where there is one. A package identifier a reader cannot resolve + # is worse than its absence. + **( { - "referenceCategory": "PACKAGE-MANAGER", - "referenceType": "purl", - "referenceLocator": component.purl, + "externalRefs": [ + { + "referenceCategory": "PACKAGE-MANAGER", + "referenceType": "purl", + "referenceLocator": component.purl, + } + ] } - ], + if component.purl != NOASSERTION + else {} + ), "attributionTexts": _attribution_texts(component, dist_info), "sourceInfo": f"{component.source_info} Reaches this wheel as: {', '.join(sorted(roles))}.", } diff --git a/scripts/sbom/sbom/report.py b/scripts/sbom/sbom/report.py index 7606cd31ce..ffa6b15408 100644 --- a/scripts/sbom/sbom/report.py +++ b/scripts/sbom/sbom/report.py @@ -26,6 +26,16 @@ } _SAFE_NAME = re.compile(r"[^A-Za-z0-9._-]") +# What a row is, so a copyright holder is not read as a package name. A vendored +# row is named for the holder its files state, because that is the only identity +# such material has -- there is no package behind it. +_KIND_PROSE = { + "source-tree": "fetched source", + "archive": "fetched archive", + "system-library": "build machine", + "vendored-source": "vendored source (named by copyright holder)", +} + _COMMIT = re.compile(r"^[0-9a-f]{40}$") @@ -265,8 +275,8 @@ def report_markdown(payload: dict) -> str: "", "## Redistributed components", "", - "| Component | Version | License | How it reaches the wheel | Evidence |", - "| --- | --- | --- | --- | --- |", + "| Component | Kind | Version | License | How it reaches the wheel | Evidence |", + "| --- | --- | --- | --- | --- | --- |", ] for item in payload["components"]: evidence = ( @@ -287,7 +297,8 @@ def report_markdown(payload: dict) -> str: ) version = item["version"] lines.append( - f"| {item['name']} | `{_short(version)}` | `{item['license_concluded']}` | " + f"| {item['name']} | {_KIND_PROSE.get(item['kind'], item['kind'])} | " + f"`{_short(version)}` | `{item['license_concluded']}` | " f"{how or '—'} | {evidence} |" ) diff --git a/tests/python/sbom/test_discovery.py b/tests/python/sbom/test_discovery.py index 721b44df0c..7a2fca5ff0 100644 --- a/tests/python/sbom/test_discovery.py +++ b/tests/python/sbom/test_discovery.py @@ -350,3 +350,31 @@ def test_a_dependency_installed_into_the_build_tree_is_not_vendored_source( holders = [key for key in discovered.components if key.startswith("vendored:")] assert not any("guennebaud" in key for key in holders), holders + + +def test_vendored_material_asserts_no_package_identifier(workspace, license_data): + """A copyright holder is not a package. + + Checked-in third-party material is identified by the copyright its files + state, because that is the only identity it has. Minting + `pkg:generic/` from it named a package that does not exist, and a + reader resolving that locator finds nothing or finds something else. + """ + from sbom import discovery, licensing + + licensing.load_corpus(license_data) + source = workspace.root / "src" / "vendor" / "widget.hpp" + source.parent.mkdir(parents=True, exist_ok=True) + source.write_text( + "// SPDX-FileCopyrightText: Copyright 2023 Upstream Widgets Ltd\n" + "// SPDX-License-Identifier: BSL-1.0\n" + ) + components, _ = discovery.discover_vendored( + workspace.root, {source}, {"Example Org"} + ) + component = next(iter(components.values())) + + assert component.kind == "vendored-source" + assert component.purl == "NOASSERTION" + # Who it came from is still stated; that much was observed. + assert component.supplier == "Organization: Upstream Widgets Ltd" From 8b03ecbd10d2a1bd4ae36e571bacaa0cbba8c694 Mon Sep 17 00:00:00 2001 From: Andrei Aristarkhov Date: Thu, 1 Oct 2026 11:38:28 -0700 Subject: [PATCH 069/101] build(sbom): do not present a copyright holder as a package name Dropping the invented purl left the package still called "Collabora Ltd", which in a `name` field reads as a product someone could obtain. It is not one: the material is a vendored header, and the holder is the only identity it has. Named for what it is -- `Vendored source (Collabora Ltd)` -- with the holder still in `supplier` and in the copyright, and the files it covers listed in `sourceInfo`. The report's Kind column no longer has to explain the name. Signed-off-by: Andrei Aristarkhov --- scripts/sbom/sbom/discovery.py | 6 +++++- scripts/sbom/sbom/report.py | 2 +- tests/python/sbom/test_discovery.py | 8 +++++++- 3 files changed, 13 insertions(+), 3 deletions(-) diff --git a/scripts/sbom/sbom/discovery.py b/scripts/sbom/sbom/discovery.py index e76c1a097c..2e4246593c 100644 --- a/scripts/sbom/sbom/discovery.py +++ b/scripts/sbom/sbom/discovery.py @@ -697,7 +697,11 @@ def discover_vendored( ) components[key] = Component( key=key, - name=names[key], + # Not the bare holder: an organisation in a `name` field reads as a + # product, and "Collabora Ltd" is not a package anyone can obtain. + # The holder is the only identity this material has, so it is named + # for what it is and who holds it; `sourceInfo` lists the files. + name=f"Vendored source ({names[key]})", kind="vendored-source", supplier=f"Organization: {names[key]}", homepage="NOASSERTION", diff --git a/scripts/sbom/sbom/report.py b/scripts/sbom/sbom/report.py index ffa6b15408..de7e4f9ad3 100644 --- a/scripts/sbom/sbom/report.py +++ b/scripts/sbom/sbom/report.py @@ -33,7 +33,7 @@ "source-tree": "fetched source", "archive": "fetched archive", "system-library": "build machine", - "vendored-source": "vendored source (named by copyright holder)", + "vendored-source": "vendored source", } diff --git a/tests/python/sbom/test_discovery.py b/tests/python/sbom/test_discovery.py index 7a2fca5ff0..800dbd60b5 100644 --- a/tests/python/sbom/test_discovery.py +++ b/tests/python/sbom/test_discovery.py @@ -154,7 +154,13 @@ def test_third_party_checked_in_here_is_still_third_party(workspace): if component.kind == "vendored-source" } - assert [c.name for c in vendored.values()] == ["Upstream Widgets Ltd"] + # Named for what it is, not as though the holder were a product. + assert [c.name for c in vendored.values()] == [ + "Vendored source (Upstream Widgets Ltd)" + ] + assert [c.supplier for c in vendored.values()] == [ + "Organization: Upstream Widgets Ltd" + ] component = next(iter(vendored.values())) assert component.license_concluded == "BSL-1.0" # The file states an identifier and carries no text of its own, so the text From 7dc47558f01f5ccc09d8d968ec76bd8f95c0b811 Mon Sep 17 00:00:00 2001 From: Andrei Aristarkhov Date: Thu, 1 Oct 2026 11:59:33 -0700 Subject: [PATCH 070/101] docs(sbom): summarise a REUSE pool in the report table OpenXR-SDK's row listed sixteen evidence files, fourteen of them its `LICENSES/` pool. A pool is a directory of reference texts kept so per-file tags resolve -- that one covers the SDK's fonts, its spec prose and a vendored jsoncpp -- so listing all of it buried `LICENSE`, which is the file the Apache-2.0 conclusion was actually read from. Grants stay in full; the pool collapses to a count and its identifiers. The JSON report and the notices packaged in the wheel still name every path, which is where a path is the thing a reader needs. What the wheel carries does not change: the pool still ships. Over-shipping a licence text is safe, and narrowing it would mean proving which of a compiled dependency's own files reached the binary. Signed-off-by: Andrei Aristarkhov --- scripts/sbom/sbom/report.py | 33 +++++++++++++++++++++------------ 1 file changed, 21 insertions(+), 12 deletions(-) diff --git a/scripts/sbom/sbom/report.py b/scripts/sbom/sbom/report.py index de7e4f9ad3..261876b948 100644 --- a/scripts/sbom/sbom/report.py +++ b/scripts/sbom/sbom/report.py @@ -6,6 +6,7 @@ from __future__ import annotations import re +from pathlib import Path from . import licensing from .inventory import Inventory @@ -279,19 +280,27 @@ def report_markdown(payload: dict) -> str: "| --- | --- | --- | --- | --- | --- |", ] for item in payload["components"]: - evidence = ( - "
".join( - f"`{record['path']}`" - + (f" → {record['identified']}" if record["identified"] else "") - + ( - " (SPDX reference text, substituted)" - if record["origin"] == licensing.SPDX_REFERENCE - else "" - ) - for record in item["evidence"] + # Grants in full; the REUSE pool summarised. A pool is a directory of + # reference texts kept so that per-file tags resolve -- OpenXR's covers + # its fonts and spec prose -- so listing all of it buries the two files + # the licence was actually read from. Every path is in the JSON report + # and in the packaged notices, which is where the paths matter. + shown = [record for record in item["evidence"] if record["kind"] != "pool"] + pooled = [record for record in item["evidence"] if record["kind"] == "pool"] + parts = [ + f"`{record['path']}`" + + (f" → {record['identified']}" if record["identified"] else "") + + ( + " (SPDX reference text, substituted)" + if record["origin"] == licensing.SPDX_REFERENCE + else "" ) - or "—" - ) + for record in shown + ] + if pooled: + names = ", ".join(sorted(Path(record["path"]).stem for record in pooled)) + parts.append(f"REUSE pool ({len(pooled)}): {names}") + evidence = "
".join(parts) or "—" how = "; ".join( _HOW_PROSE.get(value, value) for value in item["how_it_reaches_the_wheel"] ) From dbf890a65409664832a079f3e549e86b2ae94dd4 Mon Sep 17 00:00:00 2001 From: Andrei Aristarkhov Date: Thu, 1 Oct 2026 14:05:00 -0700 Subject: [PATCH 071/101] ci: assemble the release evidence without colliding on a universal wheel The transition wheel is `py3-none-any`, so every matrix entry builds it and uploads a sidecar of the same name. Copying them all into one directory made `cp` refuse, and the release publish failed with five "will not overwrite just-created" lines. They are not interchangeable: each carries an `externalDocumentRefs` entry pinning the sibling `isaaccapture` wheel that entry built, so the documents -- and therefore the wheels embedding them -- differ per matrix entry. Identical copies collapse to one file; differing ones are kept under the entry that produced them. Nothing is dropped, and nothing is silently chosen. That the same universal wheel differs by which matrix entry built it is a packaging question this step is not the place to settle. Signed-off-by: Andrei Aristarkhov --- .github/workflows/build-ubuntu.yml | 23 +++++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build-ubuntu.yml b/.github/workflows/build-ubuntu.yml index 28444c7deb..7253570ac3 100644 --- a/.github/workflows/build-ubuntu.yml +++ b/.github/workflows/build-ubuntu.yml @@ -1148,10 +1148,25 @@ jobs: shopt -s nullglob mkdir -p release-evidence - cp sbom-sidecars/*/*.spdx.json release-evidence/ - cp sbom-sidecars/*/*.licenses.json release-evidence/ - cp sbom-sidecars/*/*.licenses.md release-evidence/ - cp sbom-sidecars/*/*.build-evidence.json release-evidence/ + # A `py3-none-any` wheel is built by every matrix entry, so its sidecars + # arrive once per entry under the same name. Identical copies are one + # file; differing ones are kept apart under the entry that produced + # them, because dropping one would publish evidence for a wheel nobody + # can tell apart from the five it is not. + for sidecar in sbom-sidecars/*/*.spdx.json \ + sbom-sidecars/*/*.licenses.json \ + sbom-sidecars/*/*.licenses.md \ + sbom-sidecars/*/*.build-evidence.json; do + name="$(basename "${sidecar}")" + entry="$(basename "$(dirname "${sidecar}")")" + if [[ ! -e "release-evidence/${name}" ]]; then + cp "${sidecar}" "release-evidence/${name}" + elif cmp -s "${sidecar}" "release-evidence/${name}"; then + continue + else + cp "${sidecar}" "release-evidence/${entry}.${name}" + fi + done uv run --project scripts/sbom --python 3.12 \ isaaccapture-sbom merge-manifests \ From 15e3a615f0fb6dcdae830f1d658cc68e9169a5a6 Mon Sep 17 00:00:00 2001 From: Andrei Aristarkhov Date: Thu, 1 Oct 2026 13:40:13 -0700 Subject: [PATCH 072/101] build(sbom): keep a universal wheel universal A `py3-none-any` wheel is built by every matrix entry. Its document pinned the sibling `isaaccapture` wheel that entry produced -- namespace and SHA1 -- so the six transition wheels differed by who built them, and the release could not say which one it was publishing. `merge-manifests` refused the set, correctly. The sibling is still not an install-time unknown: version, supplier and declared licence come from its own metadata, which is the same whichever entry built it. What goes is the digest-bound reference to one entry's document. That also ends an asymmetry -- the two wheels require each other, so only one direction could ever carry such a reference, and the other already described its sibling this way. `merge_records` now collapses identical records for one filename and refuses only those that differ: one wheel described by six builds is one wheel; two wheels wearing one filename is what a consumer cannot tell apart. Signed-off-by: Andrei Aristarkhov --- scripts/sbom/sbom/build.py | 23 +++-- scripts/sbom/sbom/document.py | 34 ++----- tests/python/sbom/test_gates.py | 24 ++++- tests/python/sbom/test_wheel_sbom.py | 146 +++++---------------------- 4 files changed, 75 insertions(+), 152 deletions(-) diff --git a/scripts/sbom/sbom/build.py b/scripts/sbom/sbom/build.py index 3886ae8184..8b501bcda7 100644 --- a/scripts/sbom/sbom/build.py +++ b/scripts/sbom/sbom/build.py @@ -10,7 +10,6 @@ import os import shutil import tempfile -from collections import Counter from dataclasses import replace from pathlib import Path @@ -540,15 +539,27 @@ def merge_manifests(paths: list[Path]) -> dict: def merge_records(wheels: list[dict]) -> dict: """The manifest for a set of wheels, stamped when the set was complete.""" - names = [item["filename"] for item in wheels] - duplicates = sorted(name for name, seen in Counter(names).items() if seen > 1) - if duplicates: - raise BuildError(f"the same wheel filename was published twice: {duplicates}") + # A `py3-none-any` wheel is built by every matrix entry, so a release set + # names it once per entry. Identical records are one wheel described more + # than once; records that differ are two wheels wearing one filename, which + # is the thing a consumer cannot tell apart and this refuses to publish. + unique: dict[str, dict] = {} + conflicting: set[str] = set() + for item in wheels: + name = item["filename"] + if name in unique and unique[name] != item: + conflicting.add(name) + unique.setdefault(name, item) + if conflicting: + raise BuildError( + "the same wheel filename was published with different contents: " + + ", ".join(sorted(conflicting)) + ) return { "schema": "isaaccapture-sbom-manifest/1", "generated_at": stamped_now().strftime("%Y-%m-%dT%H:%M:%SZ"), "tool": {"name": TOOL_NAME, "version": TOOL_VERSION}, - "wheels": sorted(wheels, key=lambda item: item["filename"]), + "wheels": sorted(unique.values(), key=lambda item: item["filename"]), } diff --git a/scripts/sbom/sbom/document.py b/scripts/sbom/sbom/document.py index 0308e779ee..86a3d23275 100644 --- a/scripts/sbom/sbom/document.py +++ b/scripts/sbom/sbom/document.py @@ -509,36 +509,18 @@ def build_document( package_id, name, comment, _dependency_refs(name, group[0]) ) ) - elif sibling.get("namespace"): - # Built by this same run and already described, so the dependency - # points straight at the package in that wheel's own document. A - # local stub beside it would restate version, supplier and licence - # from the same source. - document_ref = f"DocumentRef-{licensing.spdx_safe(sibling['filename'])}" - # Two requirement lines can name one sibling -- a plain one and an - # extra-gated one -- and SPDX wants each externalDocumentId once. - entry = { - "externalDocumentId": document_ref, - "spdxDocument": sibling["namespace"], - "checksum": {"algorithm": "SHA1", "checksumValue": sibling["sha1"]}, - } - if entry not in external_documents: - external_documents.append(entry) - element = f"{document_ref}:{WHEEL_PACKAGE_ID}" - comment = ( - f"{declared} Built by this same build as {sibling['name']} " - f"{sibling['version']}; described by {sibling['filename']}." - ) else: - # Built by this same run but not yet described: two wheels can - # require each other, and a document citing another's digest cannot - # be written before it. What the wheel states about itself needs no - # document, so none of it is left unasserted. + # Built by this same run: not an install-time unknown, so the facts + # its own metadata states are recorded here. Deliberately not a + # digest-bound reference to that wheel's document -- a `py3-none-any` + # wheel is built by every matrix entry, and pinning one entry's + # sibling made the universal wheel differ by who built it. It also + # kept the two documents asymmetric, since a requirement cycle means + # only one of them could ever carry the reference. element = package_id comment = ( f"{declared} Built by this same build; its own document " - f"describes it, and is not referenced here because the two " - f"wheels require each other." + "describes its contents." ) packages.append( { diff --git a/tests/python/sbom/test_gates.py b/tests/python/sbom/test_gates.py index d20626f902..e1cb012ac9 100644 --- a/tests/python/sbom/test_gates.py +++ b/tests/python/sbom/test_gates.py @@ -222,13 +222,33 @@ def test_our_own_document_going_missing_is_rejected(built): assert any("embedded SBOM" in failure for failure in failures) -def test_duplicate_wheel_filenames_cannot_be_merged(tmp_path, built): +def test_one_wheel_described_by_several_builds_merges_to_one_record(tmp_path, built): + """A `py3-none-any` wheel is built by every matrix entry. + + Each entry's manifest names it, so a release set sees the filename once per + entry. Identical records are one wheel described more than once. + """ first = tmp_path / "a.json" second = tmp_path / "b.json" for path in (first, second): path.write_text(json.dumps(built["manifest"]), encoding="utf-8") - with pytest.raises(build_module.BuildError, match="published twice"): + merged = build_module.merge_manifests([first, second]) + names = [item["filename"] for item in merged["wheels"]] + + assert names == sorted(set(names)), "a repeated record was published twice" + + +def test_two_wheels_wearing_one_filename_are_refused(tmp_path, built): + """Same name, different bytes: the one thing a consumer cannot tell apart.""" + first = tmp_path / "a.json" + second = tmp_path / "b.json" + first.write_text(json.dumps(built["manifest"]), encoding="utf-8") + other = json.loads(json.dumps(built["manifest"])) + other["wheels"][0]["sha256"] = "0" * 64 + second.write_text(json.dumps(other), encoding="utf-8") + + with pytest.raises(build_module.BuildError, match="different contents"): build_module.merge_manifests([first, second]) diff --git a/tests/python/sbom/test_wheel_sbom.py b/tests/python/sbom/test_wheel_sbom.py index a48f7d043a..d4e716f84c 100644 --- a/tests/python/sbom/test_wheel_sbom.py +++ b/tests/python/sbom/test_wheel_sbom.py @@ -5,7 +5,6 @@ from __future__ import annotations -import hashlib import json import re import zipfile @@ -296,14 +295,17 @@ def test_components_sharing_a_licence_text_share_one_entry(workspace, license_da assert used <= defined, f"undefined after sharing: {sorted(used - defined)}" -def test_a_requirement_this_run_also_built_points_at_its_own_document( +def test_a_requirement_this_run_also_built_is_not_an_unknown( workspace, license_data, tmp_path ): """A transition wheel requires the wheel beside it; that is not an unknown. Left as an ordinary requirement it reads as something an installer resolves, - with the version, supplier and licence unasserted -- for a distribution this - same build produced and described in full. + with version, supplier and licence unasserted -- for a distribution this same + build produced. Stated from the sibling's own metadata rather than by a + digest-bound reference to its document: the two wheels require each other, so + only one direction could ever carry such a reference, and a `py3-none-any` + wheel pinning one matrix entry's sibling stops being universal. """ from sbom import cli @@ -317,9 +319,9 @@ def test_a_requirement_this_run_also_built_points_at_its_own_document( "Metadata-Version: 2.4", "Name: isaacteleop", "Version: 0.4.0", + "Author: Example Org", "License-Expression: Apache-2.0", "Requires-Dist: isaaccapture>=0.4.0", - "Author: Example Org", "", ] ).encode(), @@ -330,8 +332,6 @@ def test_a_requirement_this_run_also_built_points_at_its_own_document( ) out = workspace.root / "sbom" - # Through the CLI: it is what orders the wheels, and a document may only - # cite one that already exists. assert ( cli.main( [ @@ -353,121 +353,31 @@ def test_a_requirement_this_run_also_built_points_at_its_own_document( == 0 ) - # The two require each other, so only one document can cite the other's - # digest; neither may report the other as an install-time unknown. - sibling_doc = json.loads( - (out / f"{WHEEL_NAME.removesuffix('.whl')}.spdx.json").read_text() - ) - back = next( - package - for package in sibling_doc["packages"] - if package["name"] == "isaacteleop" - ) - assert back["versionInfo"] == "0.4.0" - assert back["licenseDeclared"] == "Apache-2.0" - assert back["supplier"] != "NOASSERTION" - assert "require each other" in back["comment"] + def sibling_row(document, name): + spdx = json.loads((out / document).read_text()) + row = next( + package + for package in spdx["packages"] + if package["name"] == name + and package["SPDXID"].startswith("SPDXRef-Package-pypi-") + ) + return spdx, row - spdx = json.loads((out / "isaacteleop-0.4.0-py3-none-any.spdx.json").read_text()) - assert not [ - package - for package in spdx["packages"] - if package["SPDXID"].startswith("SPDXRef-Package-pypi-isaaccapture") - ], "the sibling is not restated as an install-time unknown" - - ref = spdx["externalDocumentRefs"][0] - sibling = (out / f"{WHEEL_NAME.removesuffix('.whl')}.spdx.json").read_bytes() - assert ref["spdxDocument"] == json.loads(sibling)["documentNamespace"] - assert ref["checksum"] == { - "algorithm": "SHA1", - "checksumValue": hashlib.sha1(sibling).hexdigest(), # noqa: S324 - } - assert any( - item["relatedSpdxElement"] - == f"{ref['externalDocumentId']}:SPDXRef-Package-wheel" - for item in spdx["relationships"] + # Both directions state the same kind of thing, and neither pins a document. + forward, isaaccapture = sibling_row( + "isaacteleop-0.4.0-py3-none-any.spdx.json", "isaaccapture" ) - assert validate_module.check(companion) == [] - - -def _rewrite_document(wheel, mutate): - """Put a modified SBOM back into a wheel, under its own name.""" - info = wheelfile.scan(wheel) - name = wheelfile.sbom_member(info.dist_info, wheel.name) - with zipfile.ZipFile(wheel) as archive: - spdx = json.loads(archive.read(name)) - assert mutate(spdx), "nothing to mutate" - body = json.dumps(spdx, indent=2, sort_keys=True).encode() + b"\n" - staged = wheel.parent / f"staged-{wheel.name}" - wheelfile.rewrite( - wheel, - staged, - additions={}, - replacements={name: body}, - dist_info=info.dist_info, + back, isaacteleop = sibling_row( + f"{WHEEL_NAME.removesuffix('.whl')}.spdx.json", "isaacteleop" ) - staged.replace(wheel) - - -def test_a_components_terms_may_span_several_files(workspace, license_data): - """A component can state its licence in more than one file. + for spdx, row in ((forward, isaaccapture), (back, isaacteleop)): + assert "externalDocumentRefs" not in spdx + assert row["versionInfo"] != "NOASSERTION" + assert row["licenseDeclared"] == "Apache-2.0" + assert row["supplier"] != "NOASSERTION" + assert "Built by this same build" in row["comment"] - The LicenseRef then carries the join, which has no digest of its own, so the - build states one. Holding the entry to a single file's digest instead failed - a correct wheel. - """ - from sbom import licensing - - licensing.load_corpus(license_data) - root = workspace.build / "_deps" / "alpha-src" - root.mkdir(parents=True, exist_ok=True) - (root / "LICENSE").write_text(synth.PROPRIETARY_LICENSE) - (root / "COPYING").write_text(synth.PROPRIETARY_LICENSE + "\nAnd a second file.") - synth.track(workspace.root) - synth.write_wheel(workspace.wheel, synth.wheel_payload(workspace)) - - out = workspace.root / "sbom" - build_module.build(workspace.root, workspace.build, workspace.wheel, out) - - stem = WHEEL_NAME.removesuffix(".whl") - evidence_path = out / f"{stem}.build-evidence.json" - evidence = json.loads(evidence_path.read_text(encoding="utf-8")) - texts = [ - item - for item in evidence["components"]["alpha"]["evidence"] - if item["kind"] in ("grant", "pool") - ] - assert len(texts) > 1, "the component was set up to state its terms twice" - - spdx = _embedded(workspace.wheel) - joined = next( - item - for item in spdx["hasExtractedLicensingInfos"] - if "alpha" in item["licenseId"] - ) - assert joined["extractedText"].count(synth.PROPRIETARY_LICENSE) >= 1 - assert validate_module.check(workspace.wheel, build_evidence=evidence_path) == [] - - -def test_a_rewritten_licence_ref_body_is_caught_however_it_was_minted(built): - """The verbatim terms are what a LicenseRef stands for. - - A shared id belongs to no single component, which is not a reason to stop - checking it: that is the id the proprietary SDK terms travel under. - """ - swapped = "You may do whatever you like with this software." - - def rewrite(spdx): - for item in spdx["hasExtractedLicensingInfos"]: - item["extractedText"] = swapped - return bool(spdx["hasExtractedLicensingInfos"]) - - _rewrite_document(built["wheel"], rewrite) - - failures = validate_module.check( - built["wheel"], build_evidence=built["evidence_path"] - ) - assert any("not the terms the build read" in item for item in failures), failures + assert validate_module.check(companion) == [] def test_a_wheel_shipping_no_source_still_names_its_licensor(workspace, license_data): From a0e53e34cb2cbb37a705474c1be29478cc1bc600 Mon Sep 17 00:00:00 2001 From: Andrei Aristarkhov Date: Thu, 1 Oct 2026 15:18:54 -0700 Subject: [PATCH 073/101] build(sbom): judge a repeated wheel by the wheel, not by its evidence With the clock pinned, the six transition wheels a matrix produces are now byte-identical -- but their build-evidence sidecars are not, because that document describes the build, and six builds made this wheel. Comparing whole records called that a conflict and refused the release. What must agree is the artifact a consumer cannot otherwise tell apart: the filename, its digest, its size, and which member carries its SBOM. Those are compared; the first entry's evidence is the one the manifest binds, which is the set assembled beside it. That a universal wheel's evidence describes a build that put nothing in it is a real gap, and is on the open findings list rather than papered over here. Signed-off-by: Andrei Aristarkhov --- scripts/sbom/sbom/build.py | 21 +++++++++++++++------ tests/python/sbom/test_gates.py | 26 +++++++++++++++++++++++++- 2 files changed, 40 insertions(+), 7 deletions(-) diff --git a/scripts/sbom/sbom/build.py b/scripts/sbom/sbom/build.py index 8b501bcda7..a563ccae14 100644 --- a/scripts/sbom/sbom/build.py +++ b/scripts/sbom/sbom/build.py @@ -540,16 +540,25 @@ def merge_manifests(paths: list[Path]) -> dict: def merge_records(wheels: list[dict]) -> dict: """The manifest for a set of wheels, stamped when the set was complete.""" # A `py3-none-any` wheel is built by every matrix entry, so a release set - # names it once per entry. Identical records are one wheel described more - # than once; records that differ are two wheels wearing one filename, which - # is the thing a consumer cannot tell apart and this refuses to publish. + # names it once per entry. What must agree is the wheel: one filename + # standing for two different artifacts is the thing a consumer cannot tell + # apart. The sidecars may legitimately differ -- build evidence describes + # the build, and six builds produced this wheel -- so the first entry's + # evidence is the one the manifest binds, matching the set assembled beside + # it. + identity = ("filename", "sha256", "size", "sbom_in_wheel") unique: dict[str, dict] = {} conflicting: set[str] = set() for item in wheels: name = item["filename"] - if name in unique and unique[name] != item: - conflicting.add(name) - unique.setdefault(name, item) + seen = unique.get(name) + if seen is not None: + if [seen.get(key) for key in identity] != [ + item.get(key) for key in identity + ]: + conflicting.add(name) + continue + unique[name] = item if conflicting: raise BuildError( "the same wheel filename was published with different contents: " diff --git a/tests/python/sbom/test_gates.py b/tests/python/sbom/test_gates.py index e1cb012ac9..a13ab6e549 100644 --- a/tests/python/sbom/test_gates.py +++ b/tests/python/sbom/test_gates.py @@ -245,7 +245,7 @@ def test_two_wheels_wearing_one_filename_are_refused(tmp_path, built): second = tmp_path / "b.json" first.write_text(json.dumps(built["manifest"]), encoding="utf-8") other = json.loads(json.dumps(built["manifest"])) - other["wheels"][0]["sha256"] = "0" * 64 + other["wheels"][0]["sha256"] = "0" * 64 # a different wheel, same filename second.write_text(json.dumps(other), encoding="utf-8") with pytest.raises(build_module.BuildError, match="different contents"): @@ -546,3 +546,27 @@ def test_a_third_partys_licence_does_not_discharge_this_one(workspace, license_d assert build_module._declared_without_text( "Apache-2.0", {"third-party/openxr-sdk/licenses/apache-2.0.txt"} ) == ["Apache-2.0"] + + +def test_one_wheel_with_differing_evidence_merges(tmp_path, built): + """Six builds of a universal wheel produce six build-evidence sidecars. + + The wheel is the same artifact; the evidence describes the build that made + it, and those differ. A release names one evidence set, and refusing the + whole release over that would block every build of a `py3-none-any` wheel. + """ + first = tmp_path / "a.json" + second = tmp_path / "b.json" + first.write_text(json.dumps(built["manifest"]), encoding="utf-8") + other = json.loads(json.dumps(built["manifest"])) + other["wheels"][0]["sidecars"]["build_evidence"]["sha256"] = "1" * 64 + second.write_text(json.dumps(other), encoding="utf-8") + + merged = build_module.merge_manifests([first, second]) + + assert len(merged["wheels"]) == 1 + # The first entry's evidence, matching the set assembled alongside it. + assert ( + merged["wheels"][0]["sidecars"]["build_evidence"]["sha256"] + == (built["manifest"]["wheels"][0]["sidecars"]["build_evidence"]["sha256"]) + ) From 15714b1eebe5bcadc97fbbcf4e5a93b0fe333757 Mon Sep 17 00:00:00 2001 From: Andrei Aristarkhov Date: Fri, 2 Oct 2026 13:31:34 -0700 Subject: [PATCH 074/101] build(sbom): read the codemodel the reply index names The build graph was found by globbing `codemodel-v2-*.json` and taking the last match. CMake documents those names as unspecified and not to be interpreted by clients, and only *attempts* to remove replies it did not just write, so a codemodel left by an earlier configure can still be there -- and the suffix is a content hash, so sorting picks an arbitrary one of the two rather than the newer. A wheel would then be attributed from a graph that did not build it, with nothing to show for it. The index is the one file CMake does order: the new one is written before the old is deleted, and the largest name in lexicographic order is current. Enter through it and follow its reference. A configuration the codemodel does not describe is now refused for the same reason. The caller reads it from CMakeCache.txt, which already refuses to guess when the cache names none, so the two disagreeing means the reply belongs to another build; taking the first configuration instead credited a wheel to whichever one happened to be listed. Target references resolve against the codemodel rather than the reply root, which is what the format states. CMake writes both into the same directory, so this is not a change of behaviour -- it is the difference between following the format and relying on where the files happen to land. Both wheels regenerate byte-identically from a real build tree, timestamp and document namespace aside. Signed-off-by: Andrei Aristarkhov --- scripts/sbom/sbom/discovery.py | 51 +++++++++++++++++++++++++---- tests/python/sbom/synth.py | 16 +++++++++ tests/python/sbom/test_discovery.py | 21 ++++++++++++ 3 files changed, 82 insertions(+), 6 deletions(-) diff --git a/scripts/sbom/sbom/discovery.py b/scripts/sbom/sbom/discovery.py index 2e4246593c..74999e6e26 100644 --- a/scripts/sbom/sbom/discovery.py +++ b/scripts/sbom/sbom/discovery.py @@ -336,6 +336,31 @@ def check(path: Path) -> bool: return check +def _codemodel_file(reply: Path) -> Path | None: + """The current reply's codemodel, reached through its index. + + Every other reply filename is documented as unspecified and not to be + interpreted, and CMake only *attempts* to delete what it did not just + write -- so globbing for a codemodel can find one from an earlier configure, + and sorting the matches orders content hashes. The index alone carries an + order: "the one with the largest name in lexicographic order is the current + index file". + """ + if not reply.is_dir(): + return None + indexes = sorted(reply.glob("index-*.json")) + if not indexes: + return None + index = json.loads(indexes[-1].read_text(encoding="utf-8")) + for item in index.get("objects", []): + version = item.get("version") or {} + if item.get("kind") == "codemodel" and version.get("major") == 2: + # Index references are relative to the reply directory. + found = reply / item["jsonFile"] + return found if found.is_file() else None + return None + + class BuildGraph: """The link and compile graph, read from CMake's own file API. @@ -356,29 +381,43 @@ def __init__(self, build_dir: Path, config: str | None = None) -> None: def _load(self, config: str | None) -> None: reply = self.build_dir / ".cmake" / "api" / "v1" / "reply" - models = sorted(reply.glob("codemodel-v2-*.json")) if reply.is_dir() else [] - if not models: + model_file = _codemodel_file(reply) + if model_file is None: raise FileApiError( f"no CMake file API codemodel under {reply}. Configure the project " "with CMake 3.27 or newer, which is where cmake_file_api() asks for it." ) - codemodel = json.loads(models[-1].read_text(encoding="utf-8")) + codemodel = json.loads(model_file.read_text(encoding="utf-8")) source_root = Path(codemodel["paths"]["source"]) build_root = Path(codemodel["paths"]["build"]) configurations = codemodel["configurations"] chosen = next( - (item for item in configurations if item["name"] == (config or "")), - configurations[0], + (item for item in configurations if item["name"] == (config or "")), None ) + if chosen is None: + # The caller names the configuration from CMakeCache.txt, and a + # reply that does not describe it is a reply for another build -- + # taking any other credits a wheel to a graph that did not build it. + # Unnamed is the single-configuration tree the tests construct. + if config: + raise FileApiError( + f"codemodel under {reply} describes no {config!r} configuration, " + f"only {[item['name'] for item in configurations]}" + ) + chosen = configurations[0] def resolve(raw: str, root: Path) -> Path: path = Path(raw) return path if path.is_absolute() else (root / path) for entry in chosen["targets"]: - target = json.loads((reply / entry["jsonFile"]).read_text(encoding="utf-8")) + # Relative to the codemodel, which is what the format says, even + # though CMake writes it into the reply directory alongside these. + target = json.loads( + (model_file.parent / entry["jsonFile"]).read_text(encoding="utf-8") + ) includes = { resolve(item["path"], source_root).resolve() for group in target.get("compileGroups", []) diff --git a/tests/python/sbom/synth.py b/tests/python/sbom/synth.py index 158d205762..187d04f3ae 100644 --- a/tests/python/sbom/synth.py +++ b/tests/python/sbom/synth.py @@ -528,6 +528,22 @@ def _file_api_reply(root: Path, build: Path, extension: str) -> None: "configurations": [{"name": CONFIG, "targets": index}], } _write(reply / "codemodel-v2-fixture.json", json.dumps(codemodel, indent=1)) + # The collector enters through the index, as a file API client must. + _write( + reply / "index-2026-01-01T00-00-00-0000.json", + json.dumps( + { + "objects": [ + { + "kind": "codemodel", + "version": {"major": 2, "minor": 6}, + "jsonFile": "codemodel-v2-fixture.json", + } + ] + }, + indent=1, + ), + ) # ============================================================================== diff --git a/tests/python/sbom/test_discovery.py b/tests/python/sbom/test_discovery.py index 800dbd60b5..13fe64509b 100644 --- a/tests/python/sbom/test_discovery.py +++ b/tests/python/sbom/test_discovery.py @@ -5,6 +5,7 @@ from __future__ import annotations +import json from pathlib import Path import pytest @@ -128,6 +129,26 @@ def test_a_build_tree_without_a_file_api_reply_says_so(workspace): discovery.BuildGraph(workspace.build, synth.CONFIG) +def test_a_stale_codemodel_beside_the_current_one_is_not_read(workspace): + """Reply filenames are unspecified, so the index is the only thing that says + which codemodel belongs to this configure. A leftover one sorts later here.""" + reply = workspace.build / ".cmake" / "api" / "v1" / "reply" + stale = json.loads((reply / "codemodel-v2-fixture.json").read_text()) + stale["configurations"] = [{"name": synth.CONFIG, "targets": []}] + (reply / "codemodel-v2-stale.json").write_text(json.dumps(stale)) + + graph = discovery.BuildGraph(workspace.build, synth.CONFIG) + + assert graph.by_name("_ext.cpython-312-x86_64-linux-gnu.so") + + +def test_a_configuration_the_codemodel_does_not_describe_is_refused(workspace): + """The cache names the configuration; a reply that does not describe it is a + reply for another build, and attributing from it would credit the wrong one.""" + with pytest.raises(discovery.FileApiError, match="Debug"): + discovery.BuildGraph(workspace.build, "Debug") + + def test_the_graph_is_read_from_the_codemodel_not_from_generator_output(workspace): """link.txt is a Makefile-generator artifact; Ninja writes none. Reading the file API is what makes the collector work under either.""" From 4107ad27d0eb7a01526f839f96124ebb93be4a41 Mon Sep 17 00:00:00 2001 From: Andrei Aristarkhov Date: Wed, 14 Oct 2026 14:04:17 -0700 Subject: [PATCH 075/101] build(sbom): ship the license texts the root pyproject names, from both wheels The LICENSES/ glob also matched EgoPost's OpenMDW text, which no wheel carries. license-files in the root pyproject.toml is now an explicit list, and the setuptools wheel reads it at configure time instead of restating it, staging only those files. Drops the LICENSE.md copy the merge duplicated. Signed-off-by: Andrei Aristarkhov --- docs/source/references/sbom.rst | 2 +- pyproject.toml | 11 +++++- src/core/python/CMakeLists.txt | 38 ++++++++++++++++--- src/core/python/license_files.py | 33 ++++++++++++++++ src/core/python/pyproject.toml.in | 4 +- tests/python/packaging/test_wheel_contents.py | 29 ++++++++++++++ 6 files changed, 108 insertions(+), 9 deletions(-) create mode 100644 src/core/python/license_files.py diff --git a/docs/source/references/sbom.rst b/docs/source/references/sbom.rst index afaa69a08a..ff9639de2f 100644 --- a/docs/source/references/sbom.rst +++ b/docs/source/references/sbom.rst @@ -18,7 +18,7 @@ What a wheel carries ├── METADATA # License-Expression + one License-File per text below ├── licenses/ │ ├── LICENSE.md # this project's own license - │ ├── LICENSES/*.txt # texts the SPDX headers on this project's sources name + │ ├── LICENSES/.txt # the pool texts license-files in pyproject.toml names │ ├── THIRD-PARTY-NOTICES.md # component -> license -> text, with roles │ └── third-party// # its own file verbatim, or a reference text, labelled └── sboms/ diff --git a/pyproject.toml b/pyproject.toml index c32f38a01c..88dd88a8f4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -15,7 +15,16 @@ description = "Isaac Teleop" readme = "README.md" requires-python = ">=3.11" license = "Apache-2.0" -license-files = ["LICENSE.md", "LICENSES/*.txt"] +# The license texts every isaaccapture wheel ships; src/core/python/CMakeLists.txt +# reads this list for the setuptools wheel, so name a text here and nowhere else. +# Not a glob over LICENSES/: that pool also licenses components no wheel carries. +license-files = [ + "LICENSE.md", + "LICENSES/Apache-2.0.txt", + "LICENSES/BSD-3-Clause.txt", + "LICENSES/BSL-1.0.txt", + "LICENSES/MIT.txt", +] authors = [ { name = "NVIDIA" }, ] diff --git a/src/core/python/CMakeLists.txt b/src/core/python/CMakeLists.txt index 284782d119..d60ff4ea9e 100644 --- a/src/core/python/CMakeLists.txt +++ b/src/core/python/CMakeLists.txt @@ -12,6 +12,35 @@ if(NOT EXISTS "${ROBOTIC_GROUNDING_STAGED_DIR}/__init__.py") endif() message(STATUS "[grounding] bundling robotic_grounding/ from ${ROBOTIC_GROUNDING_STAGED_DIR}") +# project.license-files is stated once, in the root pyproject.toml, and both +# wheels ship that list: this one through pyproject.toml.in and the staging below. +set(_root_pyproject "${CMAKE_SOURCE_DIR}/pyproject.toml") +set(_license_files_script "${CMAKE_CURRENT_SOURCE_DIR}/license_files.py") +set_property(DIRECTORY APPEND PROPERTY CMAKE_CONFIGURE_DEPENDS + "${_root_pyproject}" "${_license_files_script}") +execute_process( + COMMAND "${Python3_EXECUTABLE}" "${_license_files_script}" "${CMAKE_SOURCE_DIR}" + OUTPUT_VARIABLE _license_files_out + ERROR_VARIABLE _license_files_err + RESULT_VARIABLE _license_files_rc + OUTPUT_STRIP_TRAILING_WHITESPACE +) +if(NOT _license_files_rc EQUAL 0) + message(FATAL_ERROR "Cannot read license-files from ${_root_pyproject}: ${_license_files_err}") +endif() +string(REPLACE "\n" ";" _license_files_out "${_license_files_out}") +list(POP_FRONT _license_files_out ISAACCAPTURE_LICENSE_FILES) +list(REMOVE_DUPLICATES _license_files_out) + +set(_license_staging_commands) +foreach(_license_file IN LISTS _license_files_out) + get_filename_component(_license_dir "${_license_file}" DIRECTORY) + list(APPEND _license_staging_commands + COMMAND ${CMAKE_COMMAND} -E make_directory "${CMAKE_BINARY_DIR}/python_package/$/${_license_dir}" + COMMAND ${CMAKE_COMMAND} -E copy "${CMAKE_SOURCE_DIR}/${_license_file}" "${CMAKE_BINARY_DIR}/python_package/$/${_license_file}" + ) +endforeach() + # Generate pyproject.toml from template with ISAAC_TELEOP_PYPROJECT_VERSION configure_file( "${CMAKE_CURRENT_SOURCE_DIR}/pyproject.toml.in" @@ -42,12 +71,11 @@ add_custom_target(python_package ALL COMMAND ${CMAKE_COMMAND} -E copy "${CMAKE_CURRENT_BINARY_DIR}/pyproject.toml" "${CMAKE_BINARY_DIR}/python_package/$/" COMMAND ${CMAKE_COMMAND} -E copy "${CMAKE_CURRENT_SOURCE_DIR}/MANIFEST.in" "${CMAKE_BINARY_DIR}/python_package/$/" COMMAND ${CMAKE_COMMAND} -E copy "${CMAKE_CURRENT_SOURCE_DIR}/setup.py" "${CMAKE_BINARY_DIR}/python_package/$/" - COMMAND ${CMAKE_COMMAND} -E copy "${CMAKE_SOURCE_DIR}/LICENSE.md" "${CMAKE_BINARY_DIR}/python_package/$/LICENSE.md" COMMAND ${CMAKE_COMMAND} -E copy "${CMAKE_SOURCE_DIR}/README.md" "${CMAKE_BINARY_DIR}/python_package/$/README.md" - # project.license-files in pyproject.toml.in names these; without them the - # wheel redistributes the project under a license it does not carry. - COMMAND ${CMAKE_COMMAND} -E copy "${CMAKE_SOURCE_DIR}/LICENSE.md" "${CMAKE_BINARY_DIR}/python_package/$/LICENSE.md" - COMMAND ${CMAKE_COMMAND} -E copy_directory "${CMAKE_SOURCE_DIR}/LICENSES" "${CMAKE_BINARY_DIR}/python_package/$/LICENSES" + # Without these the wheel redistributes the project under a license it does + # not carry. A text dropped from the list must not linger from an older build. + COMMAND ${CMAKE_COMMAND} -E rm -rf "${CMAKE_BINARY_DIR}/python_package/$/LICENSES" + ${_license_staging_commands} COMMAND ${CMAKE_COMMAND} -E copy "${CMAKE_CURRENT_SOURCE_DIR}/requirements.txt" "${CMAKE_BINARY_DIR}/python_package/$/" COMMAND ${CMAKE_COMMAND} -E copy "${CMAKE_CURRENT_SOURCE_DIR}/requirements-dev.txt" "${CMAKE_BINARY_DIR}/python_package/$/" COMMAND ${CMAKE_COMMAND} -E copy "${CMAKE_CURRENT_SOURCE_DIR}/requirements-ui.txt" "${CMAKE_BINARY_DIR}/python_package/$/" diff --git a/src/core/python/license_files.py b/src/core/python/license_files.py new file mode 100644 index 0000000000..e0e19fa335 --- /dev/null +++ b/src/core/python/license_files.py @@ -0,0 +1,33 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Print the root pyproject.toml's project.license-files for CMake. + +The first line is the TOML array, verbatim, for pyproject.toml.in; each line +after it is one file those patterns match, relative to the repository root. +""" + +import json +import sys +import tomllib +from pathlib import Path + + +def main(root: Path) -> int: + with (root / "pyproject.toml").open("rb") as handle: + patterns = tomllib.load(handle)["project"]["license-files"] + print(json.dumps(patterns)) + for pattern in patterns: + # PEP 639: a pattern that matches nothing is an error, not an empty set. + matches = sorted(path for path in root.glob(pattern) if path.is_file()) + if not matches: + print( + f"project.license-files: {pattern!r} matches no file", file=sys.stderr + ) + return 1 + for path in matches: + print(path.relative_to(root).as_posix()) + return 0 + + +if __name__ == "__main__": + sys.exit(main(Path(sys.argv[1]))) diff --git a/src/core/python/pyproject.toml.in b/src/core/python/pyproject.toml.in index 012325167b..b3552833fc 100644 --- a/src/core/python/pyproject.toml.in +++ b/src/core/python/pyproject.toml.in @@ -22,8 +22,8 @@ description = "Isaac Teleop - Teleoperation with Device I/O" readme = "README.md" requires-python = ">=3.11" license = "Apache-2.0" -# Staged beside this file by the python_package target. -license-files = ["LICENSE.md", "LICENSES/*.txt"] +# The root pyproject.toml's list, staged beside this file by python_package. +license-files = @ISAACCAPTURE_LICENSE_FILES@ authors = [ {name = "NVIDIA"} ] diff --git a/tests/python/packaging/test_wheel_contents.py b/tests/python/packaging/test_wheel_contents.py index 33c8c0674f..b1f2a9b526 100644 --- a/tests/python/packaging/test_wheel_contents.py +++ b/tests/python/packaging/test_wheel_contents.py @@ -11,6 +11,7 @@ import os import sys +import tomllib import zipfile from email.parser import BytesParser from pathlib import Path @@ -136,6 +137,34 @@ def test_capture_license_ships_in_distribution_metadata() -> None: assert not any("isaaccapture_egopost/" in name for name in archive.namelist()) +def test_wheel_ships_exactly_the_root_license_files() -> None: + """Both wheel builds take their own license texts from the root pyproject.toml.""" + root = repo_root() + with (root / "pyproject.toml").open("rb") as handle: + patterns = tomllib.load(handle)["project"]["license-files"] + expected = { + path.relative_to(root).as_posix() + for pattern in patterns + for path in root.glob(pattern) + if path.is_file() + } + with zipfile.ZipFile(_wheel()) as archive: + metadata_path = next( + name for name in archive.namelist() if name.endswith(".dist-info/METADATA") + ) + metadata = BytesParser().parsebytes(archive.read(metadata_path)) + # Third-party texts the SBOM collector appends live under other names. + own = { + name + for name in metadata.get_all("License-File", []) + if name == "LICENSE.md" or name.startswith("LICENSES/") + } + assert own == expected + licenses_dir = metadata_path.removesuffix("METADATA") + "licenses/" + for name in own: + assert archive.read(licenses_dir + name) == (root / name).read_bytes() + + @pytest.mark.skipif( not sys.platform.startswith("linux"), reason="the plugin is Linux only" ) From 27d8b57e78f6ff0837ec7b279cf58b189b877cf6 Mon Sep 17 00:00:00 2001 From: Andrei Aristarkhov Date: Wed, 14 Oct 2026 14:08:06 -0700 Subject: [PATCH 076/101] ci: leave the wheel license reports out of the build job summary They still ship in the isaaccapture-sbom-* artifact beside the SPDX documents. Signed-off-by: Andrei Aristarkhov --- .github/workflows/build-ubuntu.yml | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/.github/workflows/build-ubuntu.yml b/.github/workflows/build-ubuntu.yml index 5ac0182e88..c5b7076408 100644 --- a/.github/workflows/build-ubuntu.yml +++ b/.github/workflows/build-ubuntu.yml @@ -340,16 +340,6 @@ jobs: isaaccapture-sbom check --wheel "${wheel}" done - - name: Summarize wheel evidence (Release only) - if: matrix.build_type == 'Release' - run: | - set -euo pipefail - { - echo "## Wheel SBOM and license evidence" - echo - cat sbom/*.licenses.md - } >> "${GITHUB_STEP_SUMMARY}" - - name: Create tarball to preserve permissions if: matrix.build_type == 'Release' run: tar -cvf isaacteleop-install.tar -C install . From d53d237ded45fc2a2a4d5c6a2644a313d002d31e Mon Sep 17 00:00:00 2001 From: Andrei Aristarkhov Date: Mon, 19 Oct 2026 13:53:40 -0700 Subject: [PATCH 077/101] build(sbom): read vendored files' licences from REUSE metadata Upstream vendored Chromium's dom_code_data.inc with its licence declared only in REUSE.toml, and the collector, which read only in-file SPDX tags, refused to build. Vendored files now take their licence ids from REUSE (header, .license sidecar or REUSE.toml) and each must declare a valid SPDX licence or the build names it. The REUSE hook lints every source suffix the collector reads, and the SPDX exceptions list is now fetched. Signed-off-by: Andrei Aristarkhov --- .github/workflows/build-ubuntu.yml | 5 +- .pre-commit-config.yaml | 7 +- AGENTS.md | 6 +- deps/third_party/CMakeLists.txt | 9 +- .../third_party/fetch_license_list_data.cmake | 12 +- docs/source/references/sbom.rst | 60 ++++-- scripts/check_licenses.py | 4 +- scripts/sbom/pyproject.toml | 4 + scripts/sbom/sbom/build.py | 15 ++ scripts/sbom/sbom/discovery.py | 51 ++++- scripts/sbom/sbom/document.py | 10 +- scripts/sbom/sbom/evidence.py | 1 + scripts/sbom/sbom/licensing.py | 197 ++++++++++++++++-- scripts/sbom/sbom/report.py | 16 +- tests/python/sbom/conftest.py | 15 +- tests/python/sbom/synth.py | 102 +++++---- tests/python/sbom/test_discovery.py | 196 ++++++++++++++++- tests/python/sbom/test_gates.py | 67 ++++++ tests/python/sbom/test_licensing.py | 20 +- tests/python/sbom/test_spdx_conformance.py | 39 ++++ 20 files changed, 709 insertions(+), 127 deletions(-) diff --git a/.github/workflows/build-ubuntu.yml b/.github/workflows/build-ubuntu.yml index c5b7076408..95170875ea 100644 --- a/.github/workflows/build-ubuntu.yml +++ b/.github/workflows/build-ubuntu.yml @@ -17,8 +17,9 @@ concurrency: env: # Shared apt build dependencies. Each job installs these plus its own extras - # (ccache, clang-format, the OAK camera autotools chain). - CORE_APT_DEPS: build-essential cmake glslang-tools libegl-dev libvulkan-dev libwayland-dev libx11-dev libxcursor-dev libxext-dev libxi-dev libxinerama-dev libxkbcommon-dev libxrandr-dev patchelf pkg-config wayland-protocols + # (ccache, clang-format, the OAK camera autotools chain). libmagic1: see + # scripts/sbom/pyproject.toml. + CORE_APT_DEPS: build-essential cmake glslang-tools libegl-dev libmagic1 libvulkan-dev libwayland-dev libx11-dev libxcursor-dev libxext-dev libxi-dev libxinerama-dev libxkbcommon-dev libxrandr-dev patchelf pkg-config wayland-protocols # Pin vcpkg to an immutable commit for reproducible, supply-chain-stable CI # instead of tracking its moving default branch. Port versions are fixed by the # consuming manifest's builtin-baseline (e.g. DepthAI's vcpkg.json), so this diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 71554b4c01..cbf2b318ed 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -31,7 +31,7 @@ repos: - id: debug-statements # SPDX license headers (REUSE): require SPDX-FileCopyrightText and SPDX-License-Identifier - repo: https://github.com/fsfe/reuse-tool - rev: v6.2.0 + rev: v6.2.0 # scripts/sbom/pyproject.toml pins the same reuse to read this metadata hooks: - id: reuse-lint-file name: REUSE licenses (source files and complete EgoPost subtree) @@ -52,8 +52,9 @@ repos: name: Check copyright year (include current year) entry: python3 scripts/check_copyright_year.py --fix language: system - # Include CMakeLists.txt and Dockerfile.xxx by matching those explicitly in the regex - files: \.(py|js|ts|cpp|c|h|hpp|cxx|hxx|cmake|md|yaml|yml|sh|fbs|toml|html)$|CMakeLists\.txt$|Dockerfile(\.[A-Za-z0-9_-]+)?$ + # Include CMakeLists.txt and Dockerfile.xxx by matching those explicitly in the regex. + # Same rule as SOURCE_FILES in scripts/check_licenses.py; change both together. + files: \.(py|pyi|js|ts|cpp|cc|c|h|hpp|cxx|hxx|inc|cu|cuh|frag|vert|cmake|md|yaml|yml|sh|fbs|toml|html)$|CMakeLists\.txt$|Dockerfile(\.[A-Za-z0-9_-]+)?$ - id: check-lfs-pointers name: Ensure Git LFS-marked files are LFS pointers entry: python3 scripts/check_lfs_pointers.py diff --git a/AGENTS.md b/AGENTS.md index 48aee377cb..5369140078 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -276,11 +276,13 @@ build itself. There is **no checked-in dependency registry** and adding one woul be a second source of truth — declare a dependency the way you already would and it appears in the next wheel's inventory. -Two things fail the Release build rather than shipping a gap, and neither is +Three things fail the Release build rather than shipping a gap, and none is fixed by excluding a file or writing down an assumption: - **a wheel member the build cannot explain** — teach the collector that route; -- **a redistributed component with no license text** — obtain the terms. +- **a redistributed component with no license text** — obtain the terms; +- **a checked-in third-party file that fails the per-file REUSE gate** — record + upstream's terms in its REUSE info (sbom.rst says how). The collector is a project of its own: `uv run --project scripts/sbom isaaccapture-sbom ...`. Its dependency pins live in `scripts/sbom/pyproject.toml` diff --git a/deps/third_party/CMakeLists.txt b/deps/third_party/CMakeLists.txt index 48447e707a..d83abf5bbd 100644 --- a/deps/third_party/CMakeLists.txt +++ b/deps/third_party/CMakeLists.txt @@ -304,6 +304,7 @@ FetchContent_Declare( "-DSPDX_REPOSITORY=https://github.com/spdx/license-list-data.git" "-DSPDX_COMMIT=${SPDX_LICENSE_DATA_COMMIT}" "-DSPDX_SOURCE_DIR=${SPDX_LICENSE_DATA_SOURCE_DIR}" + "-DSPDX_SPARSE_DIR=json" -P "${CMAKE_CURRENT_LIST_DIR}/fetch_license_list_data.cmake" ) FetchContent_MakeAvailable(license_list_data) @@ -311,9 +312,11 @@ FetchContent_MakeAvailable(license_list_data) # What the SBOM collector reads, and what the sbom tests run against. set(SPDX_LICENSE_DATA_JSON_DIR "${SPDX_LICENSE_DATA_SOURCE_DIR}/json" CACHE INTERNAL "SPDX reference license texts for the SBOM collector") -if(NOT EXISTS "${SPDX_LICENSE_DATA_JSON_DIR}/licenses.json") - message(FATAL_ERROR "SPDX license list data is missing json/licenses.json after fetch") -endif() +foreach(_spdx_index licenses.json exceptions.json) + if(NOT EXISTS "${SPDX_LICENSE_DATA_JSON_DIR}/${_spdx_index}") + message(FATAL_ERROR "SPDX license list data is missing json/${_spdx_index} after fetch") + endif() +endforeach() # ============================================================================== # Chromium physical key table (vendored source, BSD-3-Clause; see chromium/README.md) diff --git a/deps/third_party/fetch_license_list_data.cmake b/deps/third_party/fetch_license_list_data.cmake index 7678eacc5a..0fcb3eb6e2 100644 --- a/deps/third_party/fetch_license_list_data.cmake +++ b/deps/third_party/fetch_license_list_data.cmake @@ -4,7 +4,7 @@ # Narrowed to the JSON the SBOM collector reads: upstream publishes every license # in several formats, none of which anything here consumes. -foreach(_required GIT_EXECUTABLE SPDX_REPOSITORY SPDX_COMMIT SPDX_SOURCE_DIR) +foreach(_required GIT_EXECUTABLE SPDX_REPOSITORY SPDX_COMMIT SPDX_SOURCE_DIR SPDX_SPARSE_DIR) if(NOT DEFINED ${_required}) message(FATAL_ERROR "${_required} is required") endif() @@ -19,6 +19,14 @@ if(EXISTS "${SPDX_SOURCE_DIR}/.git") RESULT_VARIABLE _rev_parse_result ) if(_rev_parse_result EQUAL 0 AND _current_commit STREQUAL SPDX_COMMIT) + # Same commit, but narrowed differently. SPDX_SPARSE_DIR is part of the + # download command, so changing it re-runs this. A directory, not files: + # cone mode refuses a file path once the checkout holds it. + execute_process( + COMMAND "${GIT_EXECUTABLE}" sparse-checkout set "${SPDX_SPARSE_DIR}" + WORKING_DIRECTORY "${SPDX_SOURCE_DIR}" + COMMAND_ERROR_IS_FATAL ANY + ) return() endif() endif() @@ -30,7 +38,7 @@ execute_process( COMMAND_ERROR_IS_FATAL ANY ) execute_process( - COMMAND "${GIT_EXECUTABLE}" sparse-checkout set "json/details" "json/licenses.json" + COMMAND "${GIT_EXECUTABLE}" sparse-checkout set "${SPDX_SPARSE_DIR}" WORKING_DIRECTORY "${SPDX_SOURCE_DIR}" COMMAND_ERROR_IS_FATAL ANY ) diff --git a/docs/source/references/sbom.rst b/docs/source/references/sbom.rst index ff9639de2f..05c81e2641 100644 --- a/docs/source/references/sbom.rst +++ b/docs/source/references/sbom.rst @@ -25,9 +25,9 @@ What a wheel carries └── .spdx.json # SPDX 2.3 contents inventory Every text there comes from this repository, or from a component the wheel redistributes, -or — for a component that ships no license file and states only an identifier in its source -headers — is the reference text that identifier names, beside the copyright line those -headers state. See `How licenses are determined`_. +or — for code checked into this repository, whose terms its REUSE metadata declares — is +the reference text each declared identifier names, beside the copyright line its header +states. See `How licenses are determined`_. ``.dist-info/sboms/`` is what the `binary distribution format `_ @@ -68,7 +68,8 @@ The collector is a project of its own, :code-file:`scripts/sbom/pyproject.toml`, dependencies come with it and are pinned there rather than at each call site. They are the packages that own the formats involved — ``installer`` for ``RECORD``, ``packaging`` for core metadata, ``license-expression`` for SPDX expressions, ``packageurl-python`` for -purls, ``pyelftools`` for ELF. +purls, ``pyelftools`` for ELF, ``reuse`` for REUSE metadata. ``reuse`` reads files through +libmagic, so ``build`` needs it (``libmagic1`` on Ubuntu); ``check`` does not. One command, widening with whatever you have. With just a wheel it confirms that @@ -174,11 +175,16 @@ is not for a file matched only by its path. How licenses are determined --------------------------- -Licenses are never inferred from this project's license, and never read from a list -someone maintained by hand. For each component the collector finds the license and notice -files the component itself ships — ``LICENSE``, ``COPYING``, ``BSD-LICENSE``, a ``LICENSES/`` -pool, or, where upstream ships no file at all, the license section of its README — and -matches each text against a corpus of SPDX reference texts. +Licenses are never inferred from this project's license, and never read from a list someone +maintained by hand. REUSE metadata is not such a list: it is how this repository declares +the terms of each file it holds — an in-file ``SPDX-License-Identifier``, a ``.license`` +sidecar or a ``REUSE.toml`` annotation, one declaration — and the pre-commit hook lints it +for source files. The collector reads it for third-party code checked in here; such a file +of any other kind the build reads is still held to the per-file gate below. For each fetched +component the collector finds the license and notice files the component itself ships — +``LICENSE``, ``COPYING``, ``BSD-LICENSE``, a ``LICENSES/`` pool, or, where upstream ships no +file at all, the license section of its README — and matches each text against a corpus of +SPDX reference texts. Those reference texts are a dependency like any other. :code-file:`deps/third_party/CMakeLists.txt` fetches @@ -202,9 +208,15 @@ verbatim, copyright holders and all. Substituting a reference text would convey and drop the notice — for MIT and the BSD licenses those are the same file, differing by the one line that names who licensed it to you — and it would replace the component's actual grant with our reading of it, which is what the matching is for and not what it -decides. So a reference text is substituted only where there is nothing of the component's -own to package: it states an identifier in its source headers and ships no licence file. -The document says so for every such text, in as many words, because a reader cannot tell a +decides. So a fetched component ships its own text, or, having none, blocks the build +(below). + +Third-party code checked into this repository rather than fetched is the exception: it is +identified by the copyright holder its header states, and its terms are the identifiers +REUSE gives each of its files, under REUSE's own precedence. The reference text for each +licence it names ships, and the expression keeps its structure, so a choice stays a choice; +an exception's text ships too where the expression travels as a ``LicenseRef-``, above. The +document says so for every such text, in as many words, because a reader cannot tell a template from an author's own notice by looking at it. A component can also state terms for one part of itself — a vendored asset set inside a @@ -250,10 +262,20 @@ know which license they are. Those components are listed in the report under *Li evidence gaps*. A component that states a licence nowhere at all blocks publication, because packaging the -text is the obligation the wheel has to meet. Stating one in its source headers and nowhere -else is not that case: there the reference text for the identifier those headers name ships -instead, labelled in the document as a substitution rather than as the component's own words -— it conveys the terms and names no holder, so the headers' copyright line travels with it. +text is the obligation the wheel has to meet. For code checked in here that is any one file +whose REUSE info names no licence, names one with no current SPDX reference text (a +``LicenseRef-`` among them, even beside one that has a text), states an expression that does +not parse, or names an id that is neither SPDX-listed nor a ``LicenseRef-`` or +``DocumentRef-``; an exception needs only to be listed as an exception. The build names each +such file and why, and another file of the same holder does not lend it its terms. The +remedy is to record upstream's terms as an ``SPDX-License-Identifier`` in the header, a +``.license`` sidecar or a ``REUSE.toml`` annotation. A tag that is not one — quoted in a +string, say, or prose that mentions the tag — goes between ``REUSE-IgnoreStart`` and +``REUSE-IgnoreEnd``, or is set aside by a ``.license`` sidecar or a ``REUSE.toml`` +annotation with ``precedence = "override"``. Where REUSE declares an identifier, the +reference text for it ships, labelled in the document as a substitution rather than as the +component's own words — it conveys the terms and names no holder, so the header's copyright +line travels with it. A shared library that none of those indexes explains did not come from this build, so the last thing asked is the build machine itself: resolve the library's SONAME, and take its @@ -299,8 +321,10 @@ inventory with its identity, its license and the route it took to get there. Two things do need attention: - **A component that states its license nowhere stops the build.** The fix is to obtain the - terms, not to record an assumption. Stating one only in its source headers is not that - case: the reference text for that identifier ships, labelled as a substitution. + terms, not to record an assumption. For a file checked in here, obtain the terms, then + record them as REUSE licence info naming current SPDX-listed identifiers; a stray tag + that does not parse or names no such id is set aside instead. Both remedies are under + `How licenses are determined`_. - **A new member that the build cannot explain stops the build.** That means something reached the wheel by a route the collector cannot see; teach it that route rather than excluding the file. diff --git a/scripts/check_licenses.py b/scripts/check_licenses.py index 445342fbac..e80e632a46 100644 --- a/scripts/check_licenses.py +++ b/scripts/check_licenses.py @@ -17,8 +17,10 @@ EGOPOST_LICENSE = "LicenseRef-OpenMDW-1.1" DEFAULT_LICENSE = "Apache-2.0" COPYRIGHT_HOLDER = "NVIDIA CORPORATION & AFFILIATES. All rights reserved." +# Same rule as check-copyright-year's `files:`; covers the SBOM suffixes (tested). SOURCE_FILES = re.compile( - r"\.(py|js|ts|cpp|c|h|hpp|cxx|hxx|cmake|md|yaml|yml|sh|fbs|toml|html)$" + r"\.(py|pyi|js|ts|cpp|cc|c|h|hpp|cxx|hxx|inc|cu|cuh|frag|vert|cmake|md|yaml|yml" + r"|sh|fbs|toml|html)$" r"|CMakeLists\.txt$|Dockerfile(\.[A-Za-z0-9_-]+)?$" ) diff --git a/scripts/sbom/pyproject.toml b/scripts/sbom/pyproject.toml index 2136820165..3563c66ee4 100644 --- a/scripts/sbom/pyproject.toml +++ b/scripts/sbom/pyproject.toml @@ -24,6 +24,10 @@ dependencies = [ "packageurl-python==0.17.6", # purls "packaging==26.3", # core metadata, requirements "pyelftools==0.33", # ELF dynamic section (auditwheel uses it too) + # REUSE metadata of checked-in third-party files. Keep in step with the + # reuse-tool rev in .pre-commit-config.yaml, which lints that metadata. Reads + # files through python-magic, so `build` needs libmagic1 on the runner. + "reuse==6.2.0", ] [project.optional-dependencies] diff --git a/scripts/sbom/sbom/build.py b/scripts/sbom/sbom/build.py index a563ccae14..74c95c7b4e 100644 --- a/scripts/sbom/sbom/build.py +++ b/scripts/sbom/sbom/build.py @@ -319,6 +319,21 @@ def build( + "\nThey reached the wheel by a route the collector cannot see." ) + unlicensed = sorted( + f"{path} ({key}): {reason}" + for key, component in inventory.components_present.items() + for path, reason in component.unlicensed + ) + if unlicensed: + raise BuildError( + "the REUSE licence info of these checked-in third-party files fails " + "the per-file gate:\n " + + "\n ".join(unlicensed) + + "\nRecord upstream's terms as REUSE info: a header tag, a .license " + "sidecar or a REUSE.toml annotation. For a stray tag, see 'How licenses " + "are determined' in docs/source/references/sbom.rst." + ) + missing = [ key for key, component in inventory.components_present.items() diff --git a/scripts/sbom/sbom/discovery.py b/scripts/sbom/sbom/discovery.py index 74999e6e26..d3b8fbb2af 100644 --- a/scripts/sbom/sbom/discovery.py +++ b/scripts/sbom/sbom/discovery.py @@ -127,6 +127,10 @@ class Component: # a holder and then declaring none would contradict the same document. copyright_text: str = "NOASSERTION" evidence: tuple[licensing.LicenseEvidence, ...] = () + # (path, reason) for checked-in files whose REUSE info fails the per-file + # gate. Pooling a holder's licences would otherwise lend one file the terms + # of another, or void them all. + unlicensed: tuple[tuple[str, str], ...] = () def repo_head(directory: Path) -> str | None: @@ -672,12 +676,11 @@ class _Stated: """What a source file says about itself in its header.""" holder: str - expression: str | None notice: str def _stated_by(path: Path) -> _Stated | None: - """Holder, licence and notice, from one read of the file's head.""" + """Holder and notice, from one read of the file's head.""" try: with path.open("rb") as handle: head = handle.read(4096).decode("utf-8", "replace") @@ -693,17 +696,28 @@ def _stated_by(path: Path) -> _Stated | None: notice = " ".join(match.group(0).split()).lstrip("/ *#") # The tag introduces the notice; it is not part of it. notice = re.sub(r"^SPDX-FileCopyrightText:\s*", "", notice) - return _Stated(holder, licensing.read_spdx_tag(head), notice) + return _Stated(holder, notice) def discover_vendored( repo_root: Path, candidates: set[Path], authors: set[str] ) -> tuple[dict[str, Component], dict[str, list[Path]]]: - """Group checked-in files whose copyright holder is not the project's.""" + """Group checked-in files whose copyright holder is not the project's. + + Their licences are what REUSE says of each file -- its header, a `.license` + sidecar or a REUSE.toml annotation, under REUSE's own precedence -- which is + how this repository declares the terms of every file it holds, and what its + pre-commit hook lints for source files. + """ + # Here, not at module level: only discovery needs reuse, and reuse needs libmagic. + from reuse.project import Project + + project = Project.from_directory(repo_root.resolve()) grouped: dict[str, list[Path]] = {} licences: dict[str, set[str]] = {} names: dict[str, str] = {} notices: dict[str, set[str]] = {} + unlicensed: dict[str, list[tuple[str, str]]] = {} for path in sorted(candidates): stated = _stated_by(path) @@ -715,18 +729,40 @@ def discover_vendored( grouped.setdefault(key, []).append(path) names[key] = stated.holder notices.setdefault(key, set()).add(stated.notice) - if stated.expression: - licences.setdefault(key, set()).add(stated.expression) + # The link's own path first; a link with no info of its own falls back + # to its target's, which is all the REUSE lint checks for a symlink. + under_root = path.relative_to(repo_root) + target = path.resolve() + reuse_declared = [] + for where in (project.root / under_root, target): + if where.is_relative_to(project.root) and not reuse_declared: + reuse_declared = [ + expression + for info in project.reuse_info_of(where) + for expression in info.spdx_expressions + ] + reason = licensing.why_unlicensed(reuse_declared) + if reason: + unlicensed.setdefault(key, []).append((str(under_root), reason)) + else: + licences.setdefault(key, set()).update(map(str, reuse_declared)) components: dict[str, Component] = {} for key, paths in grouped.items(): evidence = licensing.canonical_evidence(key, licences.get(key, set())) concluded, declared = licensing.expression(evidence, key) if licences.get(key): - # The headers' own expressions, structure and all. Rebuilding this + # The declared expressions, structure and all. Rebuilding this # from the reference texts the identifiers name turns `A OR B` into # `A AND B` -- a choice restated as both obligations at once. declared = licensing.combine(sorted(licences[key])) + # An id the pinned list knows and the grammar does not travels as a + # LicenseRef carrying the texts, as `licensing.expression` does. + if declared != "NOASSERTION" and not licensing.grammar_names( + licensing.identifiers_in(declared) + ): + evidence += licensing.exception_evidence(key, declared) + declared = licensing.license_ref(key) concluded = declared # Say when the list is shortened; without the marker it reads as the # whole set, and the first four by name need not be the ones that ship. @@ -761,6 +797,7 @@ def discover_vendored( licensing.fold_notices(notices.get(key, {names[key]})) ), evidence=tuple(evidence), + unlicensed=tuple(unlicensed.get(key, ())), ) return components, grouped diff --git a/scripts/sbom/sbom/document.py b/scripts/sbom/sbom/document.py index 86a3d23275..12b56fde92 100644 --- a/scripts/sbom/sbom/document.py +++ b/scripts/sbom/sbom/document.py @@ -48,6 +48,7 @@ def _attribution_texts(component: Component, dist_info: str) -> list[str]: texts = [ f"License evidence: {item.path} (sha256:{item.sha256}, {item.size} bytes, " f"{item.origin}, {item.kind}) packaged at " + # Whitespace after the path: validate.py reads it up to the first one. f"{report.packaged_license_path(dist_info, component.key, item)}" + ( " identified as " @@ -56,11 +57,10 @@ def _attribution_texts(component: Component, dist_info: str) -> list[str]: for license_id, containment, coverage in item.matches ) if item.matches - # No text of its own to package: this is the SPDX reference text for - # the identifier the component's files state about themselves, which - # is a substitution and not a reading of the component. - else f" the SPDX reference text for {item.identified}, substituted " - "because this component ships no license file" + # The SPDX reference text for an identifier the component's files + # declare, which is a substitution and not a reading of the component. + else f" — the SPDX reference text for {item.identified}, an identifier " + "REUSE declares for this component's files; not the component's own file" if item.origin == licensing.SPDX_REFERENCE else "" ) diff --git a/scripts/sbom/sbom/evidence.py b/scripts/sbom/sbom/evidence.py index 4b4a5a29ee..b54be1c8d1 100644 --- a/scripts/sbom/sbom/evidence.py +++ b/scripts/sbom/sbom/evidence.py @@ -34,6 +34,7 @@ ) # Suffixes whose files conventionally carry a REUSE header naming their holder. +# scripts/check_licenses.py lints these too; a test in tests/python/sbom ties them. _TAGGABLE_SOURCE = {".py", ".c", ".cc", ".cpp", ".h", ".hpp", ".pyi", ".sh", ".cmake"} # Cache entries that pin what a wheel contains. The rest of the cache is paths on diff --git a/scripts/sbom/sbom/licensing.py b/scripts/sbom/sbom/licensing.py index 5c6bf8cdcf..d697f3bd23 100644 --- a/scripts/sbom/sbom/licensing.py +++ b/scripts/sbom/sbom/licensing.py @@ -18,6 +18,7 @@ from license_expression import ( ExpressionError, + LicenseWithExceptionSymbol, combine_expressions, get_spdx_licensing, ) @@ -164,8 +165,8 @@ class Reference: """One SPDX licence, normalized for comparison and kept verbatim. The text is retained because it is the only authoritative copy of a licence - this build has: a component that states an identifier and ships no file of - its own has to be packaged with the text that identifier names. + this build has: a component whose terms are declared as an identifier has + to be packaged with the text that identifier names. """ name: str @@ -183,9 +184,21 @@ class CorpusError(Exception): _CORPUS: tuple[str, dict[str, Reference]] | None = None _CORPUS_META: dict[str, object] = {} +# Every licence and exception id the same list names, deprecated ones included. +_LISTED: "_Listed | None" = None # Parsing and n-gramming the whole SPDX list costs about half a second. Keyed by # directory and only ever read within one process, so a re-load is free. -_PARSED: dict[str, tuple[str, dict[str, Reference]]] = {} +_PARSED: dict[str, tuple[str, dict[str, Reference], "_Listed"]] = {} + + +@dataclass(frozen=True) +class _Listed: + """Every id the pinned list names, deprecated ones included.""" + + licences: frozenset[str] + exceptions: frozenset[str] + # Exception reference texts, from json/exceptions/.json. + exception_texts: dict[str, str] def corpus_provenance() -> dict[str, object]: @@ -196,11 +209,13 @@ def corpus_provenance() -> dict[str, object]: def load_corpus(json_dir: Path) -> str: """Load SPDX reference texts from the checkout deps/third_party fetched. - `json_dir` is the `json/` directory of spdx/license-list-data. Deprecated - identifiers are skipped: naming a component with one would be reporting a - license SPDX has withdrawn. + `json_dir` is the `json/` directory of spdx/license-list-data: the licence + reference texts, every licence and exception id the list names, and the + exceptions' texts. Deprecated licence ids are + skipped: naming a component with one would be reporting a license SPDX has + withdrawn. """ - global _CORPUS + global _CORPUS, _LISTED details = json_dir / "details" index_file = json_dir / "licenses.json" if not details.is_dir() or not index_file.is_file(): @@ -211,7 +226,7 @@ def load_corpus(json_dir: Path) -> str: cached = _PARSED.get(str(details.resolve())) if cached is not None: - version, entries = cached + version, entries, listed = cached else: entries = {} for path in sorted(details.glob("*.json")): @@ -232,12 +247,37 @@ def load_corpus(json_dir: Path) -> str: if not entries: raise CorpusError(f"no usable license texts under {details}") - version = json.loads(index_file.read_text(encoding="utf-8")).get( - "licenseListVersion", "unknown" + index = json.loads(index_file.read_text(encoding="utf-8")) + version = index.get("licenseListVersion", "unknown") + exceptions = json_dir / "exceptions.json" + try: + exception_ids = frozenset( + item["licenseExceptionId"] + for item in json.loads(exceptions.read_text(encoding="utf-8"))[ + "exceptions" + ] + ) + except (OSError, json.JSONDecodeError, KeyError, TypeError) as error: + raise CorpusError( + f"{exceptions} could not be read: {error}. Re-configure the build " + "so deps/third_party fetches it." + ) from error + texts: dict[str, str] = {} + for path in sorted((json_dir / "exceptions").glob("*.json")): + try: + payload = json.loads(path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as error: + raise CorpusError(f"{path} could not be read: {error}") from error + if payload.get("licenseExceptionText"): + texts[payload["licenseExceptionId"]] = payload["licenseExceptionText"] + listed = _Listed( + frozenset(item["licenseId"] for item in index["licenses"]), + exception_ids, + texts, ) - _PARSED[str(details.resolve())] = (version, entries) + _PARSED[str(details.resolve())] = (version, entries, listed) - _CORPUS = (version, entries) + _CORPUS, _LISTED = (version, entries), listed _CORPUS_META.clear() _CORPUS_META.update( { @@ -484,6 +524,11 @@ def expressible(license_id: str) -> bool: return False +def grammar_names(identifiers) -> bool: + """Whether the SPDX grammar this document ships with can name every id.""" + return all(expressible(item) or is_exception(item) for item in identifiers) + + def explains_the_text(matches) -> bool: """Whether the matched licences account for most of what the file says. @@ -496,6 +541,17 @@ def explains_the_text(matches) -> bool: def why_unnamed(grants: list[LicenseEvidence]) -> str: """Why terms travel verbatim rather than under an SPDX identifier.""" + if grants and all(item.origin == SPDX_REFERENCE for item in grants): + # Declared, not matched: the declaration names an id the grammar lacks. + newer = sorted( + item.identified + for item in grants + if item.identified and not grammar_names([item.identified]) + ) + return ( + f"Declares {', '.join(newer) or 'an identifier'}, newer than the SPDX " + "grammar this document ships with" + ) if not any(item.matches for item in grants): return "Matched no text in the SPDX reference corpus" partial = [item for item in grants if not explains_the_text(item.matches)] @@ -513,6 +569,101 @@ def why_unnamed(grants: list[LicenseEvidence]) -> str: return "Matched an identifier newer than the SPDX grammar this document ships with" +def _listed() -> _Listed: + if _LISTED is None: + raise CorpusError("SPDX license list data has not been loaded") + return _LISTED + + +def listed_exception(identifier: str) -> bool: + """Whether the pinned list names this id as an exception.""" + return identifier in _listed().exceptions + + +def exception_evidence(component_key: str, expression: str) -> list[LicenseEvidence]: + """The SPDX reference text for each exception an expression names.""" + texts = _listed().exception_texts + found = [] + for key in sorted(set(identifiers_in(expression)) & _listed().exceptions): + if key not in texts: + raise CorpusError( + f"json/exceptions/{key}.json is missing from the SPDX license list " + "data; re-configure the build so deps/third_party fetches it." + ) + found.append( + evidence_for( + component_key, SPDX_REFERENCE, f"{key}.txt", texts[key], "grant" + ) + ) + return found + + +def why_unlicensed(expressions) -> str | None: + """Why one vendored file's REUSE expressions fail the per-file gate, if they do. + + `expressions` are REUSE's `SpdxExpression`s, so REUSE decides what parses. + Every licence named needs a reference text: a LicenseRef beside MIT would + otherwise be defined in the document by MIT's text, and an obligation with + none ships nothing. An exception needs only to be listed as an exception. + """ + entries = corpus()[1] + licences: set[str] = set() + exceptions: set[str] = set() + for item in expressions: + try: + parsed = ( + _spdx_licensing().parse(str(item), validate=False) + if item.is_valid + else None + ) + except (ExpressionError, ValueError, TypeError): + parsed = None + if parsed is None: + return f"{str(item)!r} does not parse" + for symbol in _spdx_licensing().license_symbols(parsed, decompose=False): + if isinstance(symbol, LicenseWithExceptionSymbol): + licences.add(symbol.license_symbol.key) + exceptions.add(symbol.exception_symbol.key) + else: + licences.add(symbol.key) + listed = _listed() + # `+` is SPDX's or-later operator, not part of the id. Each slot is checked + # against its own list: after WITH, SPDX takes only a listed exception. + for ids, one, many in ( + ( + [key for key in licences if key in listed.exceptions], + "is an SPDX exception, not a licence", + "are SPDX exceptions, not licences", + ), + ( + [ + key + for key in licences + if key.removesuffix("+") not in listed.licences + and key not in listed.exceptions + and not key.startswith(("LicenseRef-", "DocumentRef-")) + ], + "is not an SPDX identifier", + "are not SPDX identifiers", + ), + ( + [key for key in exceptions if key not in listed.exceptions], + "is not an SPDX exception", + "are not SPDX exceptions", + ), + ): + if ids: + return f"{', '.join(sorted(ids))} " + (one if len(ids) == 1 else many) + textless = sorted( + key for key in licences if key not in entries or not entries[key].text + ) + if textless or not licences: + return "no current SPDX-listed licence with a reference text" + ( + f" for {', '.join(textless)}" if textless else "" + ) + return None + + def expression( evidence: list[LicenseEvidence], component_key: str | None = None ) -> tuple[str, str]: @@ -538,7 +689,7 @@ def expression( if item.origin == SPDX_REFERENCE and item.identified } ) - if identified and component_key and not all(map(expressible, identified)): + if identified and component_key and not grammar_names(identified): identified = [license_ref(component_key)] if not identified and grants and component_key: identified = [license_ref(component_key)] @@ -814,13 +965,12 @@ def combine(expressions: list[str]) -> str: def canonical_evidence( component_key: str, expressions: set[str] ) -> list[LicenseEvidence]: - """The SPDX reference text for the ids a vendored file declares about itself. + """The SPDX reference text for the ids a vendored file's REUSE info declares. - Such a component ships no licence file, so there is nothing of its own to - package verbatim and the text its identifier names is the only honest - substitute. It comes from the list this build pinned and fetched -- never - from this repository's own REUSE pool, whose copies exist to license *this* - project and carry its notices, not the component's. + The text its identifier names is what conveys those terms. It comes from the + list this build pinned and fetched -- never from this repository's own REUSE + pool, whose copies exist to license *this* project and carry its notices, + not the component's. """ found: list[LicenseEvidence] = [] seen: set[str] = set() @@ -829,10 +979,10 @@ def canonical_evidence( try: parsed = _spdx_licensing().parse(expression, validate=False) except (ExpressionError, ValueError, TypeError): - # A header tag is written by hand and can say anything -- "Public + # REUSE info is written by hand and can say anything -- "Public # domain or MIT License" is not an expression. One that will not - # parse names no reference text; the component is then reported with - # no licence evidence, which is the gap, rather than stopping here. + # parse names no reference text; the build's per-file gate names the + # file that states it, so nothing stops here. continue for token in _spdx_licensing().license_keys(parsed): reference = entries.get(token) @@ -901,7 +1051,8 @@ def is_exception(identifier: str) -> bool: """Whether an identifier is a licence exception rather than a licence. An exception modifies the licence it is attached to; the pinned list ships - their texts separately and this build fetches only the licences. + their texts separately, and one ships only where its expression travels as a + LicenseRef. """ try: parsed = _spdx_licensing().parse(identifier, validate=False) diff --git a/scripts/sbom/sbom/report.py b/scripts/sbom/sbom/report.py index 261876b948..be4dcbabca 100644 --- a/scripts/sbom/sbom/report.py +++ b/scripts/sbom/sbom/report.py @@ -165,8 +165,13 @@ def notices_markdown( lines.append( f"- Text: `{packaged}`" + ( - " — the SPDX reference text for this licence, not this " - "component's own file, which it ships none of" + " — the SPDX reference text for this " + + ( + "exception" + if licensing.listed_exception(item.identified or "") + else "licence" + ) + + ", not this component's own file" if item.origin == licensing.SPDX_REFERENCE else "" ) @@ -177,6 +182,11 @@ def notices_markdown( return "\n".join(lines) +def _lowercase_first(text: str) -> str: + """Run a sentence on; its ids keep their case.""" + return text[:1].lower() + text[1:] + + def report(inventory: Inventory, evidence_doc: dict, wheel_name: str) -> dict: """Machine-readable component-to-license report, including what is missing.""" components = [] @@ -224,7 +234,7 @@ def report(inventory: Inventory, evidence_doc: dict, wheel_name: str) -> dict: "component": key, "reason": ( "the shipped grant is carried verbatim rather than named: " - + licensing.why_unnamed(grants).lower() + + _lowercase_first(licensing.why_unnamed(grants)) ), "evidence": [item.path for item in grants], } diff --git a/tests/python/sbom/conftest.py b/tests/python/sbom/conftest.py index 25fac0d0e9..907744a2b1 100644 --- a/tests/python/sbom/conftest.py +++ b/tests/python/sbom/conftest.py @@ -5,7 +5,6 @@ from __future__ import annotations -import importlib.util import json import os from pathlib import Path @@ -16,19 +15,7 @@ from sbom import licensing -def _load(name: str, path: Path): - """Import a sibling helper by path. - - Not via `sys.path`: this directory is named `sbom`, so putting its parent on - the path would shadow the installed collector package with a namespace one. - """ - spec = importlib.util.spec_from_file_location(name, path) - module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) - return module - - -repo_root = _load( +repo_root = synth.load( "repo_paths", Path(__file__).resolve().parents[1] / "repo_paths.py" ).repo_root diff --git a/tests/python/sbom/synth.py b/tests/python/sbom/synth.py index 187d04f3ae..f4147c35e6 100644 --- a/tests/python/sbom/synth.py +++ b/tests/python/sbom/synth.py @@ -21,6 +21,7 @@ import csv import gzip import hashlib +import importlib.util import io import os import json @@ -165,6 +166,20 @@ def phdr(index: int, kind: int, offset: int, size: int) -> None: "MIT-0", "Zlib", ) +# Newer than the grammar the collector ships, which is what the tests need. +FIXTURE_EXCEPTIONS = ("rsync-linking-exception",) + + +def load(name: str, path: Path): + """Import a helper by path. + + Not via `sys.path`: this directory is named `sbom`, so putting its parent on + the path would shadow the installed collector package with a namespace one. + """ + spec = importlib.util.spec_from_file_location(name, path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module def set_license_data(path: Path) -> None: @@ -250,6 +265,15 @@ def retarget(): #pragma once """ +# Upstream's header names no identifier; this repository declares it through +# REUSE instead, as it does for Chromium's key table. +UNTAGGED_HEADER = """\ +// Copyright 2013 The Keys Authors +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. +""" +UNTAGGED_KEY = "vendored:the-keys-authors" + _PROJECT_PYPROJECT = """\ # SPDX-FileCopyrightText: Copyright (c) 2026 Example Org. All rights reserved. # SPDX-License-Identifier: Apache-2.0 @@ -284,7 +308,7 @@ def _git_checkout(root: Path, initialise: bool = True) -> None: `build/` and `dist/` are ignored here for the same reason they are in the real repository. """ - _write(root / ".gitignore", "build/\ndist/\n") + write(root / ".gitignore", "build/\ndist/\n") # Fixed identity *and* clock: a commit hashes its timestamp, the collector # records the commit, and the wheel carries that record -- so a drifting # clock makes two builds of the same project differ. @@ -311,7 +335,7 @@ def track(root: Path) -> None: _git_checkout(root, initialise=False) -def _write(path: Path, data: bytes | str) -> None: +def write(path: Path, data: bytes | str) -> None: path.parent.mkdir(parents=True, exist_ok=True) if isinstance(data, str): path.write_text(data, encoding="utf-8") @@ -322,19 +346,21 @@ def _write(path: Path, data: bytes | str) -> None: def _stage_license_data(target: Path) -> None: """Mirror what deps/third_party leaves in _deps, with just the licenses used here.""" source = license_data() - _write( - target / "licenses.json", (source / "licenses.json").read_text(encoding="utf-8") - ) + for index in ("licenses.json", "exceptions.json"): + write(target / index, (source / index).read_text(encoding="utf-8")) for license_id in FIXTURE_LICENSES: detail = source / "details" / f"{license_id}.json" if detail.is_file(): - _write(target / "details" / detail.name, detail.read_text(encoding="utf-8")) + write(target / "details" / detail.name, detail.read_text(encoding="utf-8")) + for exception_id in FIXTURE_EXCEPTIONS: + detail = source / "exceptions" / f"{exception_id}.json" + write(target / "exceptions" / detail.name, detail.read_text(encoding="utf-8")) def _gitclone_stub(deps: Path, key: str, remote: str) -> None: tmp = deps / f"{key}-subbuild" / f"{key}-populate-prefix" / "tmp" tmp.mkdir(parents=True, exist_ok=True) - _write( + write( tmp / f"{key}-populate-gitclone.cmake", " clone --no-checkout --depth 1 --no-single-branch " f'--config "advice.detachedHead=false" "{remote}" "{key}-src"\n' @@ -375,27 +401,27 @@ def create(root: Path) -> Workspace: build = workspace.build deps = build / "_deps" - _write(root / "VERSION", "0.4.x\n") - _write(root / "CMakeLists.txt", "cmake_minimum_required(VERSION 3.24)\n") - _write(root / "LICENSE.md", spdx_text("Apache-2.0")) + write(root / "VERSION", "0.4.x\n") + write(root / "CMakeLists.txt", "cmake_minimum_required(VERSION 3.24)\n") + write(root / "LICENSE.md", spdx_text("Apache-2.0")) for pool_id in ("Apache-2.0", "BSL-1.0"): - _write(root / "LICENSES" / f"{pool_id}.txt", spdx_text(pool_id)) - _write(root / "src/python/isaaccapture/__init__.py", _AUTHORED_INIT) - _write(root / "src/vendor/upstream_helper.h", _VENDORED_HEADER) - _write(root / "pyproject.toml", _PROJECT_PYPROJECT) - _write(build / "CMakeCache.txt", _CACHE) + write(root / "LICENSES" / f"{pool_id}.txt", spdx_text(pool_id)) + write(root / "src/python/isaaccapture/__init__.py", _AUTHORED_INIT) + write(root / "src/vendor/upstream_helper.h", _VENDORED_HEADER) + write(root / "pyproject.toml", _PROJECT_PYPROJECT) + write(build / "CMakeCache.txt", _CACHE) _stage_license_data(deps / "license-list-data-src" / "json") for key, (remote, license_name, license_id) in _DEPS.items(): source = deps / f"{key}-src" - _write(source / license_name, spdx_text(license_id)) - _write(source / f"{key}.cpp", f"// {key}\n") + write(source / license_name, spdx_text(license_id)) + write(source / f"{key}.cpp", f"// {key}\n") _gitclone_stub(deps, key, remote) # vendorpy ships Python that the build stages into the wheel. - _write(deps / "vendorpy-src/pkg/mod.py", _VENDOR_MODULE) - _write(deps / "vendorpy-src/pkg/assets/hand.xml", _VENDOR_ASSET_SOURCE) + write(deps / "vendorpy-src/pkg/mod.py", _VENDOR_MODULE) + write(deps / "vendorpy-src/pkg/assets/hand.xml", _VENDOR_ASSET_SOURCE) sdk_members = _sdk_tarball(root / SDK_ARCHIVE) @@ -405,44 +431,44 @@ def create(root: Path) -> Workspace: "_ext.cpython-312-x86_64-linux-gnu.so", ("libstdc++.so.6", "libc.so.6") ) gamma_bytes = elf_shared_object("libgamma_renamed.so", ("libc.so.6",)) - _write(workspace.staged / extension, extension_bytes) - _write(build / "lib/libgamma_renamed.so", gamma_bytes) - _write(build / "lib/libdelta.a", b"!\ndelta\n") - _write(build / "generated/_generated.py", _GENERATED) + write(workspace.staged / extension, extension_bytes) + write(build / "lib/libgamma_renamed.so", gamma_bytes) + write(build / "lib/libdelta.a", b"!\ndelta\n") + write(build / "generated/_generated.py", _GENERATED) # Empty, like the real PEP 561 marker: RECORD states size 0 for it, and the # bytes explain nothing, so it also stands in for any zero-length member. - _write(build / "generated/stubs/isaaccapture/py.typed", b"") + write(build / "generated/stubs/isaaccapture/py.typed", b"") # Staged tree: what the wheel is built from. # What a package manager installs into the build tree: upstream headers # carrying their own contributors' copyright lines. - _write( + write( build / "vcpkg_installed/x64-linux/include/eigen/Core.h", b"// Copyright (C) 2008 Gael Guennebaud \n", ) - _write(workspace.staged / "isaaccapture/__init__.py", _AUTHORED_INIT) - _write(workspace.staged / "isaaccapture/_generated.py", _GENERATED) - _write(workspace.staged / "isaaccapture/py.typed", b"") - _write(workspace.staged / "isaaccapture/viz/libgamma_renamed.so", gamma_bytes) - _write(workspace.staged / "isaaccapture/viz/GAMMA_LICENSE", spdx_text("Zlib")) - _write( + write(workspace.staged / "isaaccapture/__init__.py", _AUTHORED_INIT) + write(workspace.staged / "isaaccapture/_generated.py", _GENERATED) + write(workspace.staged / "isaaccapture/py.typed", b"") + write(workspace.staged / "isaaccapture/viz/libgamma_renamed.so", gamma_bytes) + write(workspace.staged / "isaaccapture/viz/GAMMA_LICENSE", spdx_text("Zlib")) + write( workspace.staged / "isaaccapture/sdk/native/libdemo.so", sdk_members["libdemo.so"], ) # patchelf drops a NEEDED entry, so the shipped bytes differ from the archive's. - _write( + write( workspace.staged / "isaaccapture/sdk/native/libextra.so", elf_shared_object("libextra.so", ("libc.so.6",)), ) - _write(workspace.staged / "vendorpy/mod.py", _VENDOR_MODULE) - _write(workspace.staged / "vendorpy/assets/hand.xml", _VENDOR_ASSET_STAGED) + write(workspace.staged / "vendorpy/mod.py", _VENDOR_MODULE) + write(workspace.staged / "vendorpy/assets/hand.xml", _VENDOR_ASSET_STAGED) _file_api_reply( root, build, (workspace.staged / extension).relative_to(build).as_posix() ) _git_checkout(root) - _write(build / "tests/unit_tests", b"\x7fELF-not-really") + write(build / "tests/unit_tests", b"\x7fELF-not-really") return workspace @@ -516,7 +542,7 @@ def _file_api_reply(root: Path, build: Path, extension: str) -> None: ], "dependencies": [{"id": dep} for dep in spec["dependencies"]], } - _write(reply / f"target-{name}-Release.json", json.dumps(payload, indent=1)) + write(reply / f"target-{name}-Release.json", json.dumps(payload, indent=1)) index.append( {"id": name, "name": name, "jsonFile": f"target-{name}-Release.json"} ) @@ -527,9 +553,9 @@ def _file_api_reply(root: Path, build: Path, extension: str) -> None: "paths": {"source": str(root), "build": str(build)}, "configurations": [{"name": CONFIG, "targets": index}], } - _write(reply / "codemodel-v2-fixture.json", json.dumps(codemodel, indent=1)) + write(reply / "codemodel-v2-fixture.json", json.dumps(codemodel, indent=1)) # The collector enters through the index, as a file API client must. - _write( + write( reply / "index-2026-01-01T00-00-00-0000.json", json.dumps( { diff --git a/tests/python/sbom/test_discovery.py b/tests/python/sbom/test_discovery.py index 13fe64509b..d3d6deaac6 100644 --- a/tests/python/sbom/test_discovery.py +++ b/tests/python/sbom/test_discovery.py @@ -184,9 +184,9 @@ def test_third_party_checked_in_here_is_still_third_party(workspace): ] component = next(iter(vendored.values())) assert component.license_concluded == "BSL-1.0" - # The file states an identifier and carries no text of its own, so the text - # comes from the SPDX list this build pinned -- never from this repository's - # own REUSE pool, whose copies license this project and carry its notices. + # The text for the identifier the file declares comes from the SPDX list this + # build pinned -- never from this repository's own REUSE pool, whose copies + # license this project and carry its notices. assert [item.path for item in component.evidence] == ["BSL-1.0.txt"] assert all(item.origin == "spdx-reference" for item in component.evidence) # The notice the component states about itself is recorded, not dropped. @@ -405,3 +405,193 @@ def test_vendored_material_asserts_no_package_identifier(workspace, license_data assert component.purl == "NOASSERTION" # Who it came from is still stated; that much was observed. assert component.supplier == "Organization: Upstream Widgets Ltd" + + +_DECLARED = { + "header": { + "src/x.h": "// Copyright 2020 Foo Inc\n// SPDX-" + + "License-Identifier: MIT OR Apache-2.0\n" + }, + "sidecar": { + "src/x.h": "// Copyright 2020 Foo Inc\n", + "src/x.h.license": "SPDX-FileCopyrightText: Copyright 2020 Foo Inc\n" + "SPDX-" + "License-Identifier: MIT OR Apache-2.0\n", + }, + "REUSE.toml": { + "src/x.h": "// Copyright 2020 Foo Inc\n", + "REUSE.toml": 'version = 1\n\n[[annotations]]\npath = "src/x.h"\n' + 'SPDX-FileCopyrightText = "Copyright 2020 Foo Inc"\n' + "SPDX-" + 'License-Identifier = "MIT OR Apache-2.0"\n', + }, +} + + +@pytest.mark.parametrize( + "files", [pytest.param(files, id=name) for name, files in _DECLARED.items()] +) +def test_a_vendored_files_terms_are_what_reuse_declares(tmp_path, files): + """A header tag, a `.license` sidecar and a REUSE.toml annotation are one + declaration; each gives the identifier as written, `OR` and all, and the + reference text for every licence it names.""" + for relative, text in files.items(): + synth.write(tmp_path / relative, text) + + components, _ = discovery.discover_vendored( + tmp_path, {tmp_path / "src/x.h"}, {"NVIDIA"} + ) + component = components["vendored:foo-inc"] + + assert component.license_concluded == "MIT OR Apache-2.0" + assert component.license_declared == "MIT OR Apache-2.0" + assert sorted(item.path for item in component.evidence) == [ + "Apache-2.0.txt", + "MIT.txt", + ] + assert all(item.origin == "spdx-reference" for item in component.evidence) + assert component.copyright_text == "Copyright 2020 Foo Inc" + + +@pytest.mark.parametrize( + ("tag", "reason"), + [ + ("", "no current SPDX-listed licence"), + ( + "// SPDX-" + "License-Identifier: LicenseRef-Foo\n", + "no current SPDX-listed licence", + ), + # MIT has a text; the LicenseRef would be defined by it in the document. + ( + "// SPDX-" + "License-Identifier: MIT AND LicenseRef-Foo\n", + "no current SPDX-listed licence with a reference text for LicenseRef-Foo", + ), + ("// SPDX-" + "License-Identifier: not ( an expression\n", "does not parse"), + # A valid tag, then one quoted in the body: REUSE reads both. + ( + "// SPDX-" + "License-Identifier: MIT\n" + 'const char *tag = "// SPDX-' + 'License-Identifier: GPL-2.0-only\\n";\n', + "does not parse", + ), + ( + "// SPDX-" + "License-Identifier: MIT AND Foo-1.0\n", + "Foo-1.0 is not an SPDX identifier", + ), + ( + "// SPDX-" + "License-Identifier: MIT AND Bar-2.0 AND Foo-1.0\n", + "Bar-2.0, Foo-1.0 are not SPDX identifiers", + ), + # `+` is SPDX's operator, so the id is known; there is just no text for it. + ( + "// SPDX-" + "License-Identifier: Apache-2.0+\n", + "no current SPDX-listed licence with a reference text", + ), + # Listed, but as a licence: the exception slot takes only exceptions. + ( + "// SPDX-" + "License-Identifier: MIT WITH Apache-2.0\n", + "Apache-2.0 is not an SPDX exception", + ), + ( + "// SPDX-" + "License-Identifier: MIT WITH LicenseRef-Foo\n", + "LicenseRef-Foo is not an SPDX exception", + ), + ( + "// SPDX-" + "License-Identifier: MIT AND rsync-linking-exception\n", + "rsync-linking-exception is an SPDX exception, not a licence", + ), + ], + ids=[ + "no-info", + "licenseref-only", + "licenseref-beside-listed", + "unparseable", + "quoted-in-body", + "unknown-id", + "unknown-ids", + "or-later", + "licence-as-exception", + "licenseref-as-exception", + "exception-as-licence", + ], +) +def test_a_file_lending_nothing_is_named_not_covered_by_its_sibling( + tmp_path, tag, reason +): + """A holder's licences are pooled for the component; a file whose own REUSE + info names a licence with no current SPDX reference text, states an + expression that does not parse, or names an id that is neither SPDX-listed + nor a LicenseRef- is still recorded, with why.""" + (tmp_path / "src").mkdir() + (tmp_path / "src/a.h").write_text( + "// Copyright 2020 Foo Inc\n// SPDX-" + "License-Identifier: MIT\n" + ) + (tmp_path / "src/b.h").write_text("// Copyright 2020 Foo Inc\n" + tag) + + components, _ = discovery.discover_vendored( + tmp_path, {tmp_path / "src/a.h", tmp_path / "src/b.h"}, {"NVIDIA"} + ) + + [(path, why)] = components["vendored:foo-inc"].unlicensed + assert path == "src/b.h" + assert reason in why + + +def test_the_reuse_hook_lints_every_source_suffix_the_collector_reads(checkout): + """The collector reads REUSE info from these; the pre-commit hook is what + lints it. Compiled suffixes no tracked-source scan names (.inc, .cu, shader + stages) reach the collector as build inputs.""" + from sbom import evidence + + hook = synth.load("check_licenses", checkout / "scripts" / "check_licenses.py") + + suffixes = evidence._TAGGABLE_SOURCE | {".inc", ".cu", ".cuh", ".frag", ".vert"} + assert not [s for s in sorted(suffixes) if not hook.SOURCE_FILES.search(f"x{s}")] + + +@pytest.mark.parametrize( + "expression", + ["BSL-1.0 WITH rsync-linking-exception", "GPL-2.0+"], + ids=["exception-newer-than-grammar", "deprecated-or-later"], +) +def test_a_listed_identifier_passes_the_per_file_gate(tmp_path, expression): + """Listed in the pinned SPDX data, whatever the grammar knows: an exception + newer than it, or an id it maps through SPDX's `+` or-later operator.""" + (tmp_path / "x.h").write_text( + "// Copyright 2020 Foo Inc\n// SPDX-" + f"License-Identifier: {expression}\n" + ) + + components, _ = discovery.discover_vendored( + tmp_path, {tmp_path / "x.h"}, {"NVIDIA"} + ) + + assert components["vendored:foo-inc"].unlicensed == () + + +def test_a_link_without_reuse_info_takes_its_targets(tmp_path): + """The REUSE lint skips symlinks and checks only the target, so a link with + no info of its own is read the same way; a link's own info still wins.""" + synth.write(tmp_path / "src/real/x.h", "// Copyright 2020 Foo Inc\n") + (tmp_path / "src/link").mkdir() + (tmp_path / "src/link/x.h").symlink_to("../real/x.h") + synth.write( + tmp_path / "REUSE.toml", + 'version = 1\n\n[[annotations]]\npath = "src/real/x.h"\n' + 'SPDX-FileCopyrightText = "Copyright 2020 Foo Inc"\n' + "SPDX-" + 'License-Identifier = "MIT"\n', + ) + + components, _ = discovery.discover_vendored( + tmp_path, {tmp_path / "src/link/x.h"}, {"NVIDIA"} + ) + + assert components["vendored:foo-inc"].unlicensed == () + assert components["vendored:foo-inc"].license_concluded == "MIT" + + synth.write( + tmp_path / "src/link/x.h.license", + "SPDX-FileCopyrightText: Copyright 2020 Foo Inc\n" + "SPDX-" + "License-Identifier: BSD-3-Clause\n", + ) + components, _ = discovery.discover_vendored( + tmp_path, {tmp_path / "src/link/x.h"}, {"NVIDIA"} + ) + + assert components["vendored:foo-inc"].license_concluded == "BSD-3-Clause" diff --git a/tests/python/sbom/test_gates.py b/tests/python/sbom/test_gates.py index a13ab6e549..299a65cfcb 100644 --- a/tests/python/sbom/test_gates.py +++ b/tests/python/sbom/test_gates.py @@ -570,3 +570,70 @@ def test_one_wheel_with_differing_evidence_merges(tmp_path, built): merged["wheels"][0]["sidecars"]["build_evidence"]["sha256"] == (built["manifest"]["wheels"][0]["sidecars"]["build_evidence"]["sha256"]) ) + + +def test_reuse_toml_declares_the_terms_of_an_untagged_vendored_file(workspace): + """A checked-in header that names no identifier, as Chromium's key table + does, under `src/vendor` so the extension that includes it ships it.""" + synth.write( + workspace.root / "src/vendor/keys/include/keys/table.h", synth.UNTAGGED_HEADER + ) + synth.write( + workspace.root / "REUSE.toml", + 'version = 1\n\n[[annotations]]\npath = "src/vendor/keys/**"\n' + 'SPDX-FileCopyrightText = "Copyright 2013 The Keys Authors"\n' + "SPDX-" + 'License-Identifier = "BSD-3-Clause"\n', + ) + synth.track(workspace.root) + _rebuild(workspace, synth.wheel_payload(workspace)) + + stem = WHEEL_NAME.removesuffix(".whl") + evidence = json.loads( + (workspace.root / "sbom" / f"{stem}.build-evidence.json").read_text() + ) + component = evidence["components"][synth.UNTAGGED_KEY] + assert component["in_this_wheel"] + assert component["license_concluded"] == "BSD-3-Clause" + assert [item["path"] for item in component["evidence"]] == ["BSD-3-Clause.txt"] + + +@pytest.mark.parametrize( + ("relative", "text", "key"), + [ + pytest.param( + "src/vendor/keys/include/keys/table.h", + synth.UNTAGGED_HEADER, + synth.UNTAGGED_KEY, + id="no-sibling", + ), + pytest.param( + "src/vendor/widgets_extra.h", + "// Copyright 2024, Upstream Widgets Ltd.\n", + "vendored:upstream-widgets-ltd", + id="no-info", + ), + pytest.param( + "src/vendor/widgets_extra.h", + "// Copyright 2024, Upstream Widgets Ltd.\n" + "// SPDX-" + "License-Identifier: LicenseRef-Widgets\n", + "vendored:upstream-widgets-ltd", + id="licenseref-only", + ), + ], +) +def test_each_vendored_file_needs_a_listed_licence_of_its_own( + workspace, relative, text, key +): + """`src/vendor/upstream_helper.h` states BSL-1.0 for its holder; a file of + another holder, or a second file of theirs, stating nothing or only a + LicenseRef with no text, is not covered by it.""" + synth.write(workspace.root / relative, text) + synth.track(workspace.root) + + with pytest.raises(build_module.BuildError) as raised: + _rebuild(workspace, synth.wheel_payload(workspace)) + + message = str(raised.value) + assert "fails the per-file gate" in message + assert f"{relative} ({key}): no current SPDX-listed licence" in message + assert "upstream_helper.h" not in message diff --git a/tests/python/sbom/test_licensing.py b/tests/python/sbom/test_licensing.py index 52cbde7d20..6b4f028ad3 100644 --- a/tests/python/sbom/test_licensing.py +++ b/tests/python/sbom/test_licensing.py @@ -5,6 +5,7 @@ from __future__ import annotations +import pytest import synth from sbom import licensing @@ -406,12 +407,25 @@ def test_a_choice_of_licences_is_not_published_as_both(license_data, tmp_path): def test_an_exception_is_not_a_licence_needing_its_own_text(license_data): """`X WITH ` must not be unsatisfiable. - The pinned list ships exception texts separately and this build fetches only - the licences, so demanding a text for the exception blocks such a - declaration with an instruction nothing in the collector can meet. + An exception's text ships only where its expression travels as a LicenseRef, + so demanding one for every exception blocks such a declaration for nothing. """ licensing.load_corpus(license_data) assert licensing.is_exception("LLVM-exception") assert licensing.is_exception("Classpath-exception-2.0") assert not licensing.is_exception("Apache-2.0") + + +@pytest.mark.parametrize("content", [None, "{not json", '{"licenses": []}']) +def test_the_exception_list_is_required(tmp_path, content): + """Without it, an exception newer than the grammar reads as an unknown id.""" + synth._stage_license_data(tmp_path) + exceptions = tmp_path / "exceptions.json" + if content is None: + exceptions.unlink() + else: + exceptions.write_text(content) + + with pytest.raises(licensing.CorpusError, match=r"exceptions\.json"): + licensing.load_corpus(tmp_path) diff --git a/tests/python/sbom/test_spdx_conformance.py b/tests/python/sbom/test_spdx_conformance.py index 15f38d836b..55518238ba 100644 --- a/tests/python/sbom/test_spdx_conformance.py +++ b/tests/python/sbom/test_spdx_conformance.py @@ -32,3 +32,42 @@ def test_relationships_survive_the_round_trip(built): assert {"DESCRIBES", "STATIC_LINK", "CONTAINS", "DEPENDS_ON"} <= kinds assert {"GENERATED_FROM", "COPY_OF"} & kinds + + +def test_an_identifier_newer_than_the_grammar_still_validates(workspace): + """A vendored file may name an exception the pinned list has and the + grammar consumers validate with does not; it travels as a LicenseRef with + its texts, and the report lists it as an evidence gap.""" + import json + + import synth + from sbom import build as build_module + + synth.write( + workspace.root / "src/vendor/rsync_helper.h", + "// Copyright 2024 Rsync Folk\n// SPDX-" + + "License-Identifier: BSL-1.0 WITH rsync-linking-exception\n", + ) + synth.track(workspace.root) + synth.write_wheel(workspace.wheel, synth.wheel_payload(workspace)) + out_dir = workspace.root / "sbom" + build_module.build(workspace.root, workspace.build, workspace.wheel, out_dir) + + stem = synth.WHEEL_NAME.removesuffix(".whl") + document = parse_file(str(out_dir / f"{stem}.spdx.json")) + assert validate_full_spdx_document(document) == [] + + report = json.loads((out_dir / f"{stem}.licenses.json").read_text()) + component = next( + item + for item in report["components"] + if item["component"] == "vendored:rsync-folk" + ) + assert component["license_concluded"] == "LicenseRef-vendored-rsync-folk" + gaps = { + item["component"]: item + for item in report["components_with_unidentified_license"] + } + # The id that caused the fallback is named, and its text ships. + assert "rsync-linking-exception" in gaps["vendored:rsync-folk"]["reason"] + assert "rsync-linking-exception.txt" in gaps["vendored:rsync-folk"]["evidence"] From 68a10af37804c6f65e1fa1fa044e586292a3ce14 Mon Sep 17 00:00:00 2001 From: Andrei Aristarkhov Date: Mon, 19 Oct 2026 16:39:13 -0700 Subject: [PATCH 078/101] build(sbom): take vendored holders from REUSE, read every covered file A third-party file whose copyright was declared only in a .license sidecar or REUSE.toml was counted as this project's own, because the collector found holders by scraping file heads, and only for a list of source suffixes kept in step with the lint by a test. Holders now follow REUSE's precedence: declared notices REUSE applies in the file's place win; otherwise the file's first copyright line still counts. Candidates are the tracked files REUSE covers plus every source the build read, so the suffix list is gone. Symlinks read as REUSE reads them. Signed-off-by: Andrei Aristarkhov --- docs/source/references/sbom.rst | 34 +- scripts/check_licenses.py | 2 +- scripts/sbom/sbom/discovery.py | 213 ++++++++-- scripts/sbom/sbom/evidence.py | 25 +- scripts/sbom/sbom/inventory.py | 8 +- scripts/sbom/sbom/report.py | 2 +- tests/python/sbom/synth.py | 23 +- tests/python/sbom/test_discovery.py | 623 ++++++++++++++++++++++++++-- tests/python/sbom/test_gates.py | 11 +- tests/python/sbom/test_licensing.py | 4 +- 10 files changed, 812 insertions(+), 133 deletions(-) diff --git a/docs/source/references/sbom.rst b/docs/source/references/sbom.rst index 05c81e2641..e9f3e258b8 100644 --- a/docs/source/references/sbom.rst +++ b/docs/source/references/sbom.rst @@ -25,9 +25,9 @@ What a wheel carries └── .spdx.json # SPDX 2.3 contents inventory Every text there comes from this repository, or from a component the wheel redistributes, -or — for code checked into this repository, whose terms its REUSE metadata declares — is -the reference text each declared identifier names, beside the copyright line its header -states. See `How licenses are determined`_. +or — for material checked into this repository, whose terms its REUSE metadata declares — +is the reference text each declared identifier names, beside the copyright notice it is +declared under. See `How licenses are determined`_. ``.dist-info/sboms/`` is what the `binary distribution format `_ @@ -179,12 +179,14 @@ Licenses are never inferred from this project's license, and never read from a l maintained by hand. REUSE metadata is not such a list: it is how this repository declares the terms of each file it holds — an in-file ``SPDX-License-Identifier``, a ``.license`` sidecar or a ``REUSE.toml`` annotation, one declaration — and the pre-commit hook lints it -for source files. The collector reads it for third-party code checked in here; such a file -of any other kind the build reads is still held to the per-file gate below. For each fetched -component the collector finds the license and notice files the component itself ships — -``LICENSE``, ``COPYING``, ``BSD-LICENSE``, a ``LICENSES/`` pool, or, where upstream ships no -file at all, the license section of its README — and matches each text against a corpus of -SPDX reference texts. +for source files. The collector reads it for every file here that REUSE covers, whatever its +type, and for every source the build compiled or included: a file identified as a third +party's (see below) is held to the per-file gate even where the hook does not lint it, and a +file naming no holder at all counts as this project's own. For each fetched component the +collector finds the license and notice files the component itself ships — ``LICENSE``, +``COPYING``, ``BSD-LICENSE``, a ``LICENSES/`` pool, or, where upstream ships no file at all, +the license section of its README — and matches each text against a corpus of SPDX reference +texts. Those reference texts are a dependency like any other. :code-file:`deps/third_party/CMakeLists.txt` fetches @@ -211,8 +213,10 @@ actual grant with our reading of it, which is what the matching is for and not w decides. So a fetched component ships its own text, or, having none, blocks the build (below). -Third-party code checked into this repository rather than fetched is the exception: it is -identified by the copyright holder its header states, and its terms are the identifiers +Third-party material checked into this repository rather than fetched is the exception: it +is identified by the copyright holder it is declared under — a ``.license`` sidecar or a +``REUSE.toml`` annotation where REUSE applies one, and the first copyright line of the file +itself unless REUSE applies those in the file's place — and its terms are the identifiers REUSE gives each of its files, under REUSE's own precedence. The reference text for each licence it names ships, and the expression keeps its structure, so a choice stays a choice; an exception's text ships too where the expression travels as a ``LicenseRef-``, above. The @@ -262,8 +266,8 @@ know which license they are. Those components are listed in the report under *Li evidence gaps*. A component that states a licence nowhere at all blocks publication, because packaging the -text is the obligation the wheel has to meet. For code checked in here that is any one file -whose REUSE info names no licence, names one with no current SPDX reference text (a +text is the obligation the wheel has to meet. For material checked in here that is any one +file whose REUSE info names no licence, names one with no current SPDX reference text (a ``LicenseRef-`` among them, even beside one that has a text), states an expression that does not parse, or names an id that is neither SPDX-listed nor a ``LicenseRef-`` or ``DocumentRef-``; an exception needs only to be listed as an exception. The build names each @@ -274,8 +278,8 @@ string, say, or prose that mentions the tag — goes between ``REUSE-IgnoreStart ``REUSE-IgnoreEnd``, or is set aside by a ``.license`` sidecar or a ``REUSE.toml`` annotation with ``precedence = "override"``. Where REUSE declares an identifier, the reference text for it ships, labelled in the document as a substitution rather than as the -component's own words — it conveys the terms and names no holder, so the header's copyright -line travels with it. +component's own words — it conveys the terms and names no holder, so the file's copyright +notice travels with it. A shared library that none of those indexes explains did not come from this build, so the last thing asked is the build machine itself: resolve the library's SONAME, and take its diff --git a/scripts/check_licenses.py b/scripts/check_licenses.py index e80e632a46..ea5bf577ef 100644 --- a/scripts/check_licenses.py +++ b/scripts/check_licenses.py @@ -17,7 +17,7 @@ EGOPOST_LICENSE = "LicenseRef-OpenMDW-1.1" DEFAULT_LICENSE = "Apache-2.0" COPYRIGHT_HOLDER = "NVIDIA CORPORATION & AFFILIATES. All rights reserved." -# Same rule as check-copyright-year's `files:`; covers the SBOM suffixes (tested). +# Same rule as check-copyright-year's `files:`. SOURCE_FILES = re.compile( r"\.(py|pyi|js|ts|cpp|cc|c|h|hpp|cxx|hxx|inc|cu|cuh|frag|vert|cmake|md|yaml|yml" r"|sh|fbs|toml|html)$" diff --git a/scripts/sbom/sbom/discovery.py b/scripts/sbom/sbom/discovery.py index d3b8fbb2af..011e3536e3 100644 --- a/scripts/sbom/sbom/discovery.py +++ b/scripts/sbom/sbom/discovery.py @@ -29,6 +29,7 @@ import tomllib import tarfile import zipfile +from collections.abc import Iterable from dataclasses import dataclass, replace from pathlib import Path @@ -601,7 +602,7 @@ def owners_under(self, directory: Path) -> set[str]: # Third-party sources vendored in this repository # ============================================================================== # Not everything third-party arrives through FetchContent. A file checked in -# here that someone else wrote says so in its REUSE header, which is the same +# here that someone else wrote says so in its REUSE info, which is the same # thing the project states about its own files -- so the copyright holder is # what separates them, not the directory it happens to sit in. @@ -612,6 +613,8 @@ def owners_under(self, directory: Path) -> set[str]: r"|^[^\w\n]*Copyright\b(?P[^\n\r]+)", re.MULTILINE, ) +# Comment markup around a copyright line: `//`, `#`, ` * `, ``, `/* ... */`. +_COMMENT_MARKUP = re.compile(r"^(?:|\*/)\s*$") _YEARS = re.compile(r"(copyright|\(c\)|©|\d{4}(\s*-\s*\d{4})?|,|\.)", re.IGNORECASE) @@ -673,46 +676,173 @@ def project_copyright(repo_root: Path) -> str: @dataclass(frozen=True) class _Stated: - """What a source file says about itself in its header.""" + """Who holds a file, and the notices that say so.""" holder: str - notice: str + notices: tuple[str, ...] + + +def _holder_in(text: str) -> str: + """The holder a copyright statement names, without years or boilerplate.""" + holder = " ".join(_YEARS.sub(" ", _COMMENT_MARKUP.sub("", text)).split()) + return holder.removesuffix(" All rights reserved").strip() + + +def _notice_line(text: str) -> str: + # The tag introduces the notice; it is not part of it. + notice = _COMMENT_MARKUP.sub("", " ".join(text.split())) + return re.sub(r"^SPDX-(?:File|Snippet)CopyrightText:\s*", "", notice) def _stated_by(path: Path) -> _Stated | None: - """Holder and notice, from one read of the file's head.""" + """Holder and notice, from the first copyright line in a text file's head.""" + from reuse.extract import HEURISTICS_CHUNK_SIZE, detect_encoding + try: with path.open("rb") as handle: - head = handle.read(4096).decode("utf-8", "replace") + raw = handle.read(4096) except OSError: return None - match = _COPYRIGHT_TAG.search(head) - if not match: + # REUSE's own text test: a PNG's `Copyright` text chunk is not a header. + encoding = detect_encoding(raw[:HEURISTICS_CHUNK_SIZE]) + if encoding is None: return None - holder = _YEARS.sub(" ", match.group("holder") or match.group("plain") or "") - holder = " ".join(holder.split()).removesuffix(" All rights reserved").strip() - if not holder: + match = _COPYRIGHT_TAG.search(raw.decode(encoding, "replace")) + if not match: return None - notice = " ".join(match.group(0).split()).lstrip("/ *#") - # The tag introduces the notice; it is not part of it. - notice = re.sub(r"^SPDX-FileCopyrightText:\s*", "", notice) - return _Stated(holder, notice) + holder = _holder_in(match.group("holder") or match.group("plain") or "") + return _Stated(holder, (_notice_line(match.group(0)),)) if holder else None + + +def _header_applies(project, used: Path, infos) -> bool: + """Whether REUSE applies the file's own text beside its declared info. + + It does where it took info from the header, and under `aggregate` whatever + it parsed -- unless a `.license` sidecar exists or any `override` applies, + the conditions `Project.reuse_info_of` tests before reading the file. + """ + from reuse.copyright import SourceType + from reuse.global_licensing import PrecedenceType + + if any(info.source_type is SourceType.FILE_HEADER for info in infos): + return True + if project.global_licensing is None or Path(f"{used}.license").exists(): + return False + found = project.global_licensing.reuse_info_of(project.relative_from_root(used)) + return PrecedenceType.AGGREGATE in found and PrecedenceType.OVERRIDE not in found + + +def _held_by(project, used: Path, infos, authors: set[str]) -> _Stated | None: + """Who other than the project holds a file, as a `_Stated`, if anyone does. + + A sidecar or REUSE.toml notice naming someone else makes the file theirs, + under the first such holder by name. The file's first copyright line counts + as well unless REUSE applied declared notices that all name a holder in the + file's place: a sidecar, `override`, or `closest` where REUSE found nothing + in the file. Then the file is ours if those all name the project. + """ + from reuse.copyright import SourceType + + def ours(holder: str) -> bool: + return any(author.lower() in holder.lower() for author in authors) + + # (holder, notice, whether it names the project), once per declared notice. + declared = [ + ( + _holder_in(notice.name), + _notice_line(notice.to_string(original=True)), + ours(notice.name), + ) + for info in infos + if info.source_type is not SourceType.FILE_HEADER + for notice in info.copyright_notices + ] + set_aside = ( + bool(declared) + and all(holder for holder, _, _ in declared) + and not _header_applies(project, used, infos) + ) + stated = None if set_aside else _stated_by(used) + scraped = stated if stated is not None and not ours(stated.holder) else None + foreign = [(holder, line) for holder, line, mine in declared if holder and not mine] + if not foreign: + return scraped + holder = min(foreign, key=lambda item: (item[0].casefold(), item))[0] + lines = [line for _, line in foreign] + list(scraped.notices if scraped else ()) + return _Stated(holder, tuple(lines)) + + +def _covered(project, path: Path, *, built: bool) -> bool: + """Whether REUSE covers a file. + + For a file the build read, only what REUSE excludes by name counts: a link + is judged by its target, and a submodule, a meson subproject or a + VCS-ignored file it compiled still ships. + """ + from reuse.covered_files import is_path_ignored + + if not built and path.is_symlink(): + return False + real = path.resolve() + if not real.is_relative_to(project.root) or not real.is_file(): + return False + # With no VCS strategy REUSE checks neither ignore rules nor submodules. + return not any( + is_path_ignored( + item, + include_submodules=project.include_submodules, + include_meson_subprojects=built or project.include_meson_subprojects, + vcs_strategy=None if built else project.vcs_strategy, + ) + for item in (real, *real.parents) + if item != project.root and item.is_relative_to(project.root) + ) + + +def _info_for(project, own: Path, target: Path): + """REUSE info for a file's holder and for its licence, and the holder's path. + + A link's own declarations are its sidecar and the annotations on its path. + Where they give a field, REUSE's view of the link decides it, as for a file + in its place; otherwise REUSE's view of the target, which applies the + target's precedence to the bytes they share. + """ + from reuse.copyright import SourceType + + infos = project.reuse_info_of(own) + if target == own: + return infos, infos, own + mine = [info for info in infos if info.source_type is not SourceType.FILE_HEADER] + theirs = project.reuse_info_of(target) + licensed = infos if any(info.spdx_expressions for info in mine) else theirs + if any(info.copyright_notices for info in mine): + return infos, licensed, own + return theirs, licensed, target def discover_vendored( - repo_root: Path, candidates: set[Path], authors: set[str] + repo_root: Path, + candidates: set[Path], + authors: set[str], + *, + tracked: Iterable[str], ) -> tuple[dict[str, Component], dict[str, list[Path]]]: """Group checked-in files whose copyright holder is not the project's. - Their licences are what REUSE says of each file -- its header, a `.license` - sidecar or a REUSE.toml annotation, under REUSE's own precedence -- which is - how this repository declares the terms of every file it holds, and what its - pre-commit hook lints for source files. + Reads the build inputs in `candidates` and the `tracked` files REUSE covers + (`_covered`); `tracked` is required, as without it tracked data and Python + go unread. Licences are REUSE's; holders are `_held_by`'s. """ # Here, not at module level: only discovery needs reuse, and reuse needs libmagic. from reuse.project import Project project = Project.from_directory(repo_root.resolve()) + built = {path for path in candidates if _covered(project, path, built=True)} + candidates = built | { + path + for path in (repo_root / item for item in tracked) + if _covered(project, path, built=False) + } grouped: dict[str, list[Path]] = {} licences: dict[str, set[str]] = {} names: dict[str, str] = {} @@ -720,27 +850,19 @@ def discover_vendored( unlicensed: dict[str, list[tuple[str, str]]] = {} for path in sorted(candidates): - stated = _stated_by(path) + under_root = path.relative_to(repo_root) + own = project.root / under_root + infos, licensed, used = _info_for(project, own, path.resolve()) + stated = _held_by(project, used, infos, authors) if stated is None: continue - if any(author.lower() in stated.holder.lower() for author in authors): - continue + reuse_declared = [ + expression for info in licensed for expression in info.spdx_expressions + ] key = f"vendored:{licensing.spdx_safe(stated.holder).lower()}" grouped.setdefault(key, []).append(path) names[key] = stated.holder - notices.setdefault(key, set()).add(stated.notice) - # The link's own path first; a link with no info of its own falls back - # to its target's, which is all the REUSE lint checks for a symlink. - under_root = path.relative_to(repo_root) - target = path.resolve() - reuse_declared = [] - for where in (project.root / under_root, target): - if where.is_relative_to(project.root) and not reuse_declared: - reuse_declared = [ - expression - for info in project.reuse_info_of(where) - for expression in info.spdx_expressions - ] + notices.setdefault(key, set()).update(stated.notices) reason = licensing.why_unlicensed(reuse_declared) if reason: unlicensed.setdefault(key, []).append((str(under_root), reason)) @@ -764,9 +886,14 @@ def discover_vendored( evidence += licensing.exception_evidence(key, declared) declared = licensing.license_ref(key) concluded = declared - # Say when the list is shortened; without the marker it reads as the - # whole set, and the first four by name need not be the ones that ship. - named = sorted(str(item.relative_to(repo_root)) for item in paths) + # What the build read first, so a shortened list names what ships; say + # when it is shortened, or it reads as the whole set. + named = [ + name + for _, name in sorted( + (item not in built, str(item.relative_to(repo_root))) for item in paths + ) + ] relative = ", ".join(named[:4]) + ( f", and {len(named) - 4} more" if len(named) > 4 else "" ) @@ -781,15 +908,15 @@ def discover_vendored( supplier=f"Organization: {names[key]}", homepage="NOASSERTION", # No purl: this is third-party material identified by the copyright - # its files state, and there is no package to locate. Minting - # `pkg:generic/` named a package that does not exist, from - # an organisation rather than from anything shipped. + # holder its REUSE info or header names; there is no package to + # locate. Minting `pkg:generic/` named a package that does + # not exist, from an organisation rather than from anything shipped. purl="NOASSERTION", version="NOASSERTION", download_location="NOASSERTION", source_info=( - f"Checked into this repository rather than fetched; states its own " - f"copyright. Files: {relative}." + f"Checked into this repository rather than fetched; its REUSE info " + f"or header names its copyright holder. Files: {relative}." ), license_concluded=concluded, license_declared=declared, diff --git a/scripts/sbom/sbom/evidence.py b/scripts/sbom/sbom/evidence.py index b54be1c8d1..c24dc77fcd 100644 --- a/scripts/sbom/sbom/evidence.py +++ b/scripts/sbom/sbom/evidence.py @@ -33,10 +33,6 @@ EMPTY_SHA256, ) -# Suffixes whose files conventionally carry a REUSE header naming their holder. -# scripts/check_licenses.py lints these too; a test in tests/python/sbom ties them. -_TAGGABLE_SOURCE = {".py", ".c", ".cc", ".cpp", ".h", ".hpp", ".pyi", ".sh", ".cmake"} - # Cache entries that pin what a wheel contains. The rest of the cache is paths on # the builder. _CACHE_KEYS = re.compile( @@ -272,9 +268,6 @@ def discover(repo_root: Path, build_dir: Path) -> Discovery: if len(owners) == 1: staged_component_dirs[display] = owners.pop() - # Third-party code checked in here rather than fetched: the candidates are - # what the build actually compiles or includes, so nothing outside the build - # is inspected. ownership = Ownership() for root in sorted(deps_dir.glob("*-src")): if root.is_dir(): @@ -287,19 +280,13 @@ def discover(repo_root: Path, build_dir: Path) -> Discovery: if root.is_dir(): ownership.register_root(root, root.name.removesuffix("-build")) - # Every tracked file that can carry a REUSE header, plus whatever the build - # compiled. Scanning only compiled sources missed a third party whose code - # reaches the wheel as Python: nothing compiles it, so nothing looked at it, - # and the component was reported as contributing no file. - candidates: set[Path] = { - repo_root / item - for item in tracked - if Path(item).suffix.lower() in _TAGGABLE_SOURCE - } - candidates.update(graph.repo_inputs(repo_root, build_dir)) - + # Third-party material checked in here rather than fetched. Tracked files as + # well as build inputs: Python or data reaching the wheel is compiled by nothing. vendored, owned_paths = discover_vendored( - repo_root, candidates, project_authors(repo_root) + repo_root, + graph.repo_inputs(repo_root, build_dir), + project_authors(repo_root), + tracked=tracked, ) components.update(vendored) for key, paths in owned_paths.items(): diff --git a/scripts/sbom/sbom/inventory.py b/scripts/sbom/sbom/inventory.py index 24ae996bd3..360f6ae950 100644 --- a/scripts/sbom/sbom/inventory.py +++ b/scripts/sbom/sbom/inventory.py @@ -346,10 +346,10 @@ def _from_repo_file( ) -> Attribution: """A file this repository holds -- and, where it is not ours, whose. - Checked-in third-party code states its own copyright, which is how the - vendored scan found it. Reporting the member as plain repository source - would drop that holder, and the component would be reported as - contributing nothing to a wheel that ships its file. + Checked-in third-party material declares its own copyright holder, + which is how the vendored scan found it. Reporting the member as plain + repository source would drop that holder, and the component would be + reported as contributing nothing to a wheel that ships its file. """ owner = self.ownership.owner_of(self.repo_root / repo_file.removeprefix("./")) return Attribution( diff --git a/scripts/sbom/sbom/report.py b/scripts/sbom/sbom/report.py index be4dcbabca..76f76702a7 100644 --- a/scripts/sbom/sbom/report.py +++ b/scripts/sbom/sbom/report.py @@ -28,7 +28,7 @@ _SAFE_NAME = re.compile(r"[^A-Za-z0-9._-]") # What a row is, so a copyright holder is not read as a package name. A vendored -# row is named for the holder its files state, because that is the only identity +# row is named for the holder its REUSE info or header names, the only identity # such material has -- there is no package behind it. _KIND_PROSE = { "source-tree": "fetched source", diff --git a/tests/python/sbom/synth.py b/tests/python/sbom/synth.py index f4147c35e6..5484e66b7a 100644 --- a/tests/python/sbom/synth.py +++ b/tests/python/sbom/synth.py @@ -275,7 +275,7 @@ def retarget(): UNTAGGED_KEY = "vendored:the-keys-authors" _PROJECT_PYPROJECT = """\ -# SPDX-FileCopyrightText: Copyright (c) 2026 Example Org. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 [project] @@ -330,6 +330,27 @@ def _git_checkout(root: Path, initialise: bool = True) -> None: run("commit", "-q", "--allow-empty", "-m", "synthetic project") +def annotation( + path: str, + holder: str | None = None, + *, + licence: str | None = "MIT", + precedence: str = "closest", + year: int = 2020, +) -> str: + """One REUSE.toml annotation; it states no copyright where `holder` is None.""" + lines = ["[[annotations]]", f'path = "{path}"', f'precedence = "{precedence}"'] + if holder is not None: + lines.append(f'SPDX-FileCopyrightText = "Copyright {year} {holder}"') + if licence is not None: + lines.append("SPDX-" + f'License-Identifier = "{licence}"') + return "\n".join(lines) + "\n" + + +def reuse_toml(*annotations: str) -> str: + return "version = 1\n\n" + "\n".join(annotations) + + def track(root: Path) -> None: """Commit whatever a test added after the project was generated.""" _git_checkout(root, initialise=False) diff --git a/tests/python/sbom/test_discovery.py b/tests/python/sbom/test_discovery.py index d3d6deaac6..9dee044c72 100644 --- a/tests/python/sbom/test_discovery.py +++ b/tests/python/sbom/test_discovery.py @@ -6,6 +6,8 @@ from __future__ import annotations import json +import subprocess +import zlib from pathlib import Path import pytest @@ -163,9 +165,9 @@ def test_the_graph_is_read_from_the_codemodel_not_from_generator_output(workspac def test_third_party_checked_in_here_is_still_third_party(workspace): """Not everything third-party arrives through FetchContent. A file checked - into the repository that someone else wrote says so in its copyright line, - and that is what separates it from the project's own code -- not the - directory it sits in.""" + into the repository that someone else wrote says so in its REUSE info, and + that is what separates it from the project's own code -- not the directory + it sits in.""" from sbom import evidence as evidence_module space = evidence_module.discover(workspace.root, workspace.build) @@ -382,8 +384,8 @@ def test_a_dependency_installed_into_the_build_tree_is_not_vendored_source( def test_vendored_material_asserts_no_package_identifier(workspace, license_data): """A copyright holder is not a package. - Checked-in third-party material is identified by the copyright its files - state, because that is the only identity it has. Minting + Checked-in third-party material is identified by the copyright holder its + REUSE info or header names, because that is the only identity it has. Minting `pkg:generic/` from it named a package that does not exist, and a reader resolving that locator finds nothing or finds something else. """ @@ -397,7 +399,7 @@ def test_vendored_material_asserts_no_package_identifier(workspace, license_data "// SPDX-License-Identifier: BSL-1.0\n" ) components, _ = discovery.discover_vendored( - workspace.root, {source}, {"Example Org"} + workspace.root, {source}, {"Example Org"}, tracked=() ) component = next(iter(components.values())) @@ -407,6 +409,50 @@ def test_vendored_material_asserts_no_package_identifier(workspace, license_data assert component.supplier == "Organization: Upstream Widgets Ltd" +def _discover(root: Path, *candidates: str): + """Discover over build inputs alone, as `discover_vendored` sees them.""" + components, _ = discovery.discover_vendored( + root, {root / item for item in candidates}, {"NVIDIA"}, tracked=() + ) + return components + + +def _vendored(root: Path, files: dict[str, str | bytes]): + """Discover over files as a checkout tracking all of them would.""" + for relative, data in files.items(): + synth.write(root / relative, data) + components, _ = discovery.discover_vendored( + root, set(), {"NVIDIA"}, tracked=sorted(files) + ) + return components + + +def _link(root: Path, link: str, target: str) -> None: + (root / link).parent.mkdir(parents=True, exist_ok=True) + (root / link).symlink_to(target) + + +def _notices(components) -> dict[str, set[str]]: + return {key: set(c.copyright_text.splitlines()) for key, c in components.items()} + + +_LICENCE_TAG = "SPDX-" + "License-Identifier: MIT" +_IGNORED_BAR = "// REUSE-IgnoreStart\n// Copyright 2020 Bar Ltd\n// REUSE-IgnoreEnd\n" +# Third-party headers, by what REUSE can read of them. +_HEADERS = { + "copyright-and-licence": f"// Copyright 2020 Bar Ltd\n// {_LICENCE_TAG}\n", + "copyright-only": "// Copyright 2020 Bar Ltd\n", + "unparsed-and-licence": f"// Copyright: 2020 Bar Ltd\n// {_LICENCE_TAG}\n", + "unparsed-only": "// Copyright: 2020 Bar Ltd\n", + "ignored-and-licence": f"{_IGNORED_BAR}// {_LICENCE_TAG}\n", + "ignored-only": _IGNORED_BAR, + "ignored-and-contributor": f"{_IGNORED_BAR}// SPDX-FileContributor: Someone\n", +} +_BAR = {"vendored:bar-ltd": {"Copyright 2020 Bar Ltd"}} +_BAR_UNPARSED = {"vendored:bar-ltd": {"Copyright: 2020 Bar Ltd"}} +_FOO = "Copyright 2020 Foo Inc" + + _DECLARED = { "header": { "src/x.h": "// Copyright 2020 Foo Inc\n// SPDX-" @@ -419,9 +465,9 @@ def test_vendored_material_asserts_no_package_identifier(workspace, license_data }, "REUSE.toml": { "src/x.h": "// Copyright 2020 Foo Inc\n", - "REUSE.toml": 'version = 1\n\n[[annotations]]\npath = "src/x.h"\n' - 'SPDX-FileCopyrightText = "Copyright 2020 Foo Inc"\n' - "SPDX-" + 'License-Identifier = "MIT OR Apache-2.0"\n', + "REUSE.toml": synth.reuse_toml( + synth.annotation("src/x.h", "Foo Inc", licence="MIT OR Apache-2.0") + ), }, } @@ -436,10 +482,7 @@ def test_a_vendored_files_terms_are_what_reuse_declares(tmp_path, files): for relative, text in files.items(): synth.write(tmp_path / relative, text) - components, _ = discovery.discover_vendored( - tmp_path, {tmp_path / "src/x.h"}, {"NVIDIA"} - ) - component = components["vendored:foo-inc"] + component = _discover(tmp_path, "src/x.h")["vendored:foo-inc"] assert component.license_concluded == "MIT OR Apache-2.0" assert component.license_declared == "MIT OR Apache-2.0" @@ -525,27 +568,13 @@ def test_a_file_lending_nothing_is_named_not_covered_by_its_sibling( ) (tmp_path / "src/b.h").write_text("// Copyright 2020 Foo Inc\n" + tag) - components, _ = discovery.discover_vendored( - tmp_path, {tmp_path / "src/a.h", tmp_path / "src/b.h"}, {"NVIDIA"} - ) + components = _discover(tmp_path, "src/a.h", "src/b.h") [(path, why)] = components["vendored:foo-inc"].unlicensed assert path == "src/b.h" assert reason in why -def test_the_reuse_hook_lints_every_source_suffix_the_collector_reads(checkout): - """The collector reads REUSE info from these; the pre-commit hook is what - lints it. Compiled suffixes no tracked-source scan names (.inc, .cu, shader - stages) reach the collector as build inputs.""" - from sbom import evidence - - hook = synth.load("check_licenses", checkout / "scripts" / "check_licenses.py") - - suffixes = evidence._TAGGABLE_SOURCE | {".inc", ".cu", ".cuh", ".frag", ".vert"} - assert not [s for s in sorted(suffixes) if not hook.SOURCE_FILES.search(f"x{s}")] - - @pytest.mark.parametrize( "expression", ["BSL-1.0 WITH rsync-linking-exception", "GPL-2.0+"], @@ -558,40 +587,544 @@ def test_a_listed_identifier_passes_the_per_file_gate(tmp_path, expression): "// Copyright 2020 Foo Inc\n// SPDX-" + f"License-Identifier: {expression}\n" ) - components, _ = discovery.discover_vendored( - tmp_path, {tmp_path / "x.h"}, {"NVIDIA"} - ) - - assert components["vendored:foo-inc"].unlicensed == () + assert _discover(tmp_path, "x.h")["vendored:foo-inc"].unlicensed == () def test_a_link_without_reuse_info_takes_its_targets(tmp_path): """The REUSE lint skips symlinks and checks only the target, so a link with no info of its own is read the same way; a link's own info still wins.""" synth.write(tmp_path / "src/real/x.h", "// Copyright 2020 Foo Inc\n") - (tmp_path / "src/link").mkdir() - (tmp_path / "src/link/x.h").symlink_to("../real/x.h") + _link(tmp_path, "src/link/x.h", "../real/x.h") synth.write( tmp_path / "REUSE.toml", - 'version = 1\n\n[[annotations]]\npath = "src/real/x.h"\n' - 'SPDX-FileCopyrightText = "Copyright 2020 Foo Inc"\n' - "SPDX-" + 'License-Identifier = "MIT"\n', + synth.reuse_toml(synth.annotation("src/real/x.h", "Foo Inc")), ) - components, _ = discovery.discover_vendored( - tmp_path, {tmp_path / "src/link/x.h"}, {"NVIDIA"} - ) + component = _discover(tmp_path, "src/link/x.h")["vendored:foo-inc"] - assert components["vendored:foo-inc"].unlicensed == () - assert components["vendored:foo-inc"].license_concluded == "MIT" + assert component.unlicensed == () + assert component.license_concluded == "MIT" synth.write( tmp_path / "src/link/x.h.license", "SPDX-FileCopyrightText: Copyright 2020 Foo Inc\n" "SPDX-" + "License-Identifier: BSD-3-Clause\n", ) + component = _discover(tmp_path, "src/link/x.h")["vendored:foo-inc"] + + assert component.license_concluded == "BSD-3-Clause" + + +@pytest.mark.parametrize( + "files", + [ + pytest.param( + { + "src/x.h": "#pragma once\n", + "src/x.h.license": f"SPDX-FileCopyrightText: {_FOO}\n{_LICENCE_TAG}\n", + }, + id="sidecar", + ), + pytest.param( + { + "assets/icon.svg": "\n", + "REUSE.toml": synth.reuse_toml(synth.annotation("assets/*", "Foo Inc")), + }, + id="reuse-toml-svg", + ), + pytest.param( + { + "assets/data.json": "{}\n", + "REUSE.toml": synth.reuse_toml(synth.annotation("assets/*", "Foo Inc")), + }, + id="reuse-toml-json", + ), + ], +) +def test_a_holder_declared_beside_the_file_is_read(tmp_path, files): + """A file of any type whose copyright only REUSE metadata states is still + someone else's; the sidecar and REUSE.toml are not candidates themselves.""" + component = _vendored(tmp_path, files)["vendored:foo-inc"] + + [covered] = [name for name in files if not name.endswith((".license", ".toml"))] + assert covered in component.source_info + assert component.license_concluded == "MIT" + assert component.copyright_text == _FOO + + +@pytest.mark.parametrize( + ("header", "holder", "precedence", "expected"), + [ + pytest.param( + "copyright-and-licence", + "Foo Inc", + "override", + {"vendored:foo-inc": {_FOO}}, + id="override-outranks-the-header", + ), + pytest.param( + "copyright-only", None, "override", _BAR, id="override-licence-only" + ), + pytest.param( + "copyright-and-licence", + "Foo Inc", + "closest", + _BAR, + id="closest-takes-a-full-header-first", + ), + pytest.param( + "unparsed-only", + "NVIDIA CORPORATION", + "closest", + {}, + id="closest-stands-in-for-a-file-reuse-finds-nothing-in", + ), + pytest.param( + "unparsed-and-licence", + "NVIDIA CORPORATION", + "closest", + _BAR_UNPARSED, + id="closest-beside-a-licence-tag", + ), + pytest.param( + "ignored-and-contributor", + "NVIDIA CORPORATION", + "closest", + _BAR, + id="closest-beside-a-contributor-line", + ), + *( + pytest.param( + header, + "NVIDIA CORPORATION", + "aggregate", + _BAR_UNPARSED if header.startswith("unparsed") else _BAR, + id=f"aggregate-project-{header}", + ) + for header in ( + "copyright-and-licence", + "copyright-only", + "unparsed-and-licence", + "ignored-and-licence", + "unparsed-only", + "ignored-only", + ) + ), + pytest.param( + "copyright-and-licence", + "Foo Inc", + "aggregate", + {"vendored:foo-inc": {_FOO, "Copyright 2020 Bar Ltd"}}, + id="aggregate-both-give-notices", + ), + ], +) +def test_a_header_counts_where_reuse_applies_it_beside_an_annotation( + tmp_path, header, holder, precedence, expected +): + """`override` sets the file aside; `aggregate` applies it beside the + annotation whatever REUSE parsed; `closest` applies it where REUSE took + anything from it, and stands in for it where REUSE found nothing.""" + toml = synth.reuse_toml(synth.annotation("src/x.h", holder, precedence=precedence)) + components = _vendored(tmp_path, {"src/x.h": _HEADERS[header], "REUSE.toml": toml}) + + assert _notices(components) == expected + + +def test_declared_notices_naming_the_project_keep_the_file_first_party(tmp_path): + """The header's holder does not decide where a sidecar names the project.""" + components = _vendored( + tmp_path, + { + "src/x.h": _HEADERS["copyright-only"], + "src/x.h.license": "SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA " + "CORPORATION & AFFILIATES. All rights reserved.\n" + "SPDX-" + "License-Identifier: Apache-2.0\n", + }, + ) + + assert components == {} + + +def test_a_file_with_several_declared_holders_is_grouped_by_one(tmp_path): + """Grouped under the first other holder by name, whatever its case, with + every notice but the project's as its copyright.""" + components = _vendored( + tmp_path, + { + "src/x.h": "#pragma once\n", + "src/x.h.license": "SPDX-FileCopyrightText: 2020 Zed Corp\n" + "SPDX-FileCopyrightText: 2026 NVIDIA CORPORATION\n" + "SPDX-FileCopyrightText: 2021 acme Inc\n" + _LICENCE_TAG + "\n", + }, + ) + + assert _notices(components) == { + "vendored:acme-inc": {"2020 Zed Corp", "2021 acme Inc"} + } + + +def _png_with_text(keyword: bytes, text: bytes) -> bytes: + def chunk(kind: bytes, data: bytes) -> bytes: + body = kind + data + return len(data).to_bytes(4, "big") + body + zlib.crc32(body).to_bytes(4, "big") + + header = (1).to_bytes(4, "big") * 2 + bytes([8, 0, 0, 0, 0]) + pixels = zlib.compress(b"\x00\x00") + return ( + b"\x89PNG\r\n\x1a\n" + + chunk(b"IHDR", header) + + chunk(b"tEXt", keyword + b"\x00" + text) + + chunk(b"IDAT", pixels) + + chunk(b"IEND", b"") + ) + + +def test_a_binary_file_is_not_scraped_for_a_holder(tmp_path): + """A PNG's `Copyright` text chunk is not a header.""" + png = _png_with_text(b"Copyright", b"\nCopyright 2020 Foo Inc\n") + assert discovery._COPYRIGHT_TAG.search(png.decode("utf-8", "replace")) + + assert _vendored(tmp_path, {"assets/logo.png": png}) == {} + + +def test_license_files_and_the_pool_are_not_candidates(tmp_path): + """REUSE does not cover `LICENSES/` or `LICENSE*`/`COPYING*` files, and the + holder one names is its template's or its licensor's.""" + upstream = "Copyright (c) 2020 Foo Inc\n\nPermission is hereby granted\n" + components = _vendored( + tmp_path, + { + "LICENSE": upstream, + "COPYING.txt": upstream, + "deps/foo/LICENSE.md": upstream, + "LICENSES/MIT.txt": synth.spdx_text("MIT"), + }, + ) + + assert components == {} + + +def test_a_build_input_reuse_does_not_cover_is_not_a_candidate(tmp_path): + """An include directory's licence text and sidecar are not sources.""" + synth.write(tmp_path / "inc/x.h", "// Copyright 2020 Foo Inc\n") + synth.write(tmp_path / "inc/x.h.license", f"SPDX-FileCopyrightText: {_FOO}\n") + synth.write(tmp_path / "inc/LICENSE", "Copyright (c) 2020 Bar Ltd\n") + + components = _discover(tmp_path, "inc/x.h", "inc/x.h.license", "inc/LICENSE") + + assert list(components) == ["vendored:foo-inc"] + assert "Files: inc/x.h." in components["vendored:foo-inc"].source_info + + +@pytest.mark.parametrize( + "relative", + ["subprojects/foo/x.h", "gen/x.h", "sub/x.h"], + ids=["meson-subproject", "vcs-ignored", "submodule"], +) +def test_a_build_input_the_project_does_not_cover_is_still_read(tmp_path, relative): + """A compiled file ships whether or not REUSE lints it; a tracked one there is + held to REUSE's coverage.""" + subprocess.run(["git", "init", "-q", str(tmp_path)], check=True) + synth.write(tmp_path / ".gitignore", "gen/\n") + synth.write(tmp_path / ".gitmodules", '[submodule "sub"]\n\tpath = sub\n') + synth.write(tmp_path / relative, "// Copyright 2020 Foo Inc\n") + + assert list(_discover(tmp_path, relative)) == ["vendored:foo-inc"] + components, _ = discovery.discover_vendored( + tmp_path, set(), {"NVIDIA"}, tracked=[relative] + ) + assert components == {} + + +def test_the_files_the_build_read_are_named_first(tmp_path): + for name in ("src/a.h", "src/z.h"): + synth.write(tmp_path / name, "// Copyright 2020 Foo Inc\n") + + components, _ = discovery.discover_vendored( + tmp_path, {tmp_path / "src/z.h"}, {"NVIDIA"}, tracked=["src/a.h", "src/z.h"] + ) + + assert "Files: src/z.h, src/a.h." in components["vendored:foo-inc"].source_info + + +def test_a_tracked_non_code_file_reaches_discovery(workspace, license_data): + """Through the build's own entry point, not only `discover_vendored`.""" + from sbom import evidence, licensing + + licensing.load_corpus(license_data) + synth.write(workspace.root / "assets/vendor/icon.svg", "\n") + synth.write( + workspace.root / "REUSE.toml", + synth.reuse_toml(synth.annotation("assets/vendor/**", "Foo Inc")), + ) + synth.track(workspace.root) + + discovered = evidence.discover(workspace.root, workspace.build) + + assert ( + "assets/vendor/icon.svg" + in discovered.components["vendored:foo-inc"].source_info + ) + + +def test_a_copyright_string_in_a_first_party_body_names_no_holder(tmp_path): + """REUSE reports every copyright line in a file; only the head's first one + is the file's own.""" + components = _vendored( + tmp_path, + { + "src/x.py": "# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA " + "CORPORATION & AFFILIATES. All rights reserved.\n" + "# SPDX-" + "License-Identifier: Apache-2.0\n\n" + 'NOTICE = """\nCopyright 2019 Someone Else\n"""\n', + }, + ) + + assert components == {} + + +@pytest.mark.parametrize( + ("files", "expected"), + [ + pytest.param( + { + "src/x.h": _HEADERS["copyright-only"], + "src/x.h.license": _LICENCE_TAG + "\n", + }, + _BAR, + id="sidecar-licence-only", + ), + pytest.param( + {"src/x.h": _HEADERS["unparsed-and-licence"]}, + _BAR_UNPARSED, + id="line-reuse-does-not-parse", + ), + pytest.param( + {"src/x.h": _HEADERS["ignored-and-licence"]}, + _BAR, + id="inside-an-ignore-block", + ), + pytest.param( + { + "src/x.h": _HEADERS["copyright-only"], + "src/x.h.license": f"SPDX-FileCopyrightText: 2020\n{_LICENCE_TAG}\n", + }, + _BAR, + id="declared-notice-naming-nobody", + ), + ], +) +def test_without_a_declared_holder_the_head_still_decides(tmp_path, files, expected): + """Where REUSE declares no copyright holder, a text file's first copyright + line names it, whether or not REUSE read that line.""" + assert _notices(_vendored(tmp_path, files)) == expected + + +def test_the_head_is_read_in_the_encoding_it_is_written_in(tmp_path): + header = _HEADERS["copyright-and-licence"].encode("utf-16") + + assert list(_vendored(tmp_path, {"src/x.h": header})) == ["vendored:bar-ltd"] + + +@pytest.mark.parametrize("tag", ["SPDX-FileCopyrightText", "SPDX-SnippetCopyrightText"]) +def test_a_declared_notice_is_published_as_written(tmp_path, tag): + components = _vendored( + tmp_path, + { + "src/x.h": "#pragma once\n", + "src/x.h.license": f"{tag}: 2020,2021 Foo Inc\n{_LICENCE_TAG}\n", + }, + ) + + assert components["vendored:foo-inc"].copyright_text == "2020,2021 Foo Inc" + + +@pytest.mark.parametrize( + "sidecar", + [f"SPDX-FileCopyrightText: 2026 NVIDIA CORPORATION\n{_LICENCE_TAG}\n", ""], + ids=["declaring", "empty"], +) +def test_an_aggregate_annotation_beside_a_sidecar_sets_the_file_aside( + tmp_path, sidecar +): + """REUSE reads the sidecar in the file's place, aggregate or not, and + whether or not the sidecar says anything.""" + toml = synth.annotation("src/x.h", "NVIDIA CORPORATION", precedence="aggregate") + components = _vendored( + tmp_path, + { + "src/x.h": _HEADERS["copyright-and-licence"], + "src/x.h.license": sidecar, + "REUSE.toml": synth.reuse_toml(toml), + }, + ) + + assert components == {} + + +def test_a_nested_override_sets_the_file_aside_beneath_an_aggregate(tmp_path): + """REUSE stops reading the file at any override, whatever lies above it.""" + nvidia = "NVIDIA CORPORATION" + components = _vendored( + tmp_path, + { + "src/x.h": _HEADERS["copyright-and-licence"], + "REUSE.toml": synth.reuse_toml( + synth.annotation("**", nvidia, precedence="aggregate") + ), + "src/REUSE.toml": synth.reuse_toml( + synth.annotation("**", nvidia, precedence="override") + ), + }, + ) + + assert components == {} + + +def test_a_link_takes_its_targets_holder_where_it_has_none(tmp_path): + synth.write(tmp_path / "vendor/x.h", f"// {_LICENCE_TAG}\n") + synth.write( + tmp_path / "vendor/x.h.license", "SPDX-FileCopyrightText: 2020 Foo Inc\n" + ) + _link(tmp_path, "src/x.h", "../vendor/x.h") + + component = _discover(tmp_path, "src/x.h")["vendored:foo-inc"] + + # REUSE gives the target no licence: its sidecar sets the header aside. + assert component.license_concluded == "NOASSERTION" + assert component.unlicensed + + +def test_a_link_takes_its_own_licence_and_its_targets_holder(tmp_path): + synth.write(tmp_path / "vendor/x.h", _HEADERS["copyright-and-licence"]) + _link(tmp_path, "src/x.h", "../vendor/x.h") + synth.write( + tmp_path / "src/x.h.license", "SPDX-" + "License-Identifier: BSD-3-Clause\n" + ) + + component = _discover(tmp_path, "src/x.h")["vendored:bar-ltd"] + + assert component.license_concluded == "BSD-3-Clause" + + +@pytest.mark.parametrize( + ("sidecar", "expected"), + [ + pytest.param( + "SPDX-FileCopyrightText: 2026 NVIDIA CORPORATION\n" + "SPDX-" + "License-Identifier: Apache-2.0\n", + {}, + id="target-sidecar-names-the-project", + ), + pytest.param( + f"SPDX-FileCopyrightText: {_FOO}\nSPDX-" + + "License-Identifier: BSD-3-Clause\n", + {"vendored:foo-inc": "BSD-3-Clause"}, + id="target-sidecar-names-another", + ), + ], +) +def test_a_link_reads_its_target_as_reuse_does(tmp_path, sidecar, expected): + """The bytes a link shares with its target are read under the target's + precedence: a header the target's sidecar sets aside decides nothing.""" + synth.write(tmp_path / "vendor/x.h", _HEADERS["copyright-and-licence"]) + synth.write(tmp_path / "vendor/x.h.license", sidecar) + _link(tmp_path, "src/x.h", "../vendor/x.h") + + components = _discover(tmp_path, "src/x.h") + + assert {key: c.license_concluded for key, c in components.items()} == expected + + +@pytest.mark.parametrize("through", ["file", "link"]) +@pytest.mark.parametrize( + ("header", "holder", "precedence", "expected"), + [ + pytest.param( + "copyright-and-licence", + "Foo Inc", + "aggregate", + {"vendored:foo-inc": "Apache-2.0 AND MIT"}, + id="aggregate", + ), + pytest.param( + "unparsed-only", + "NVIDIA CORPORATION", + "aggregate", + {"vendored:bar-ltd": "Apache-2.0"}, + id="aggregate-unparsed", + ), + pytest.param( + "unparsed-and-licence", + "NVIDIA CORPORATION", + "closest", + {"vendored:bar-ltd": "MIT"}, + id="closest", + ), + ], +) +def test_a_link_with_its_own_annotation_reads_as_a_file_in_its_place( + tmp_path, through, header, holder, precedence, expected +): + annotation = synth.annotation( + "src/**", holder, licence="Apache-2.0", precedence=precedence + ) + synth.write(tmp_path / "REUSE.toml", synth.reuse_toml(annotation)) + if through == "link": + synth.write(tmp_path / "vendor/x.h", _HEADERS[header]) + _link(tmp_path, "src/x.h", "../vendor/x.h") + else: + synth.write(tmp_path / "src/x.h", _HEADERS[header]) + + components = _discover(tmp_path, "src/x.h") + + assert {key: c.license_concluded for key, c in components.items()} == expected + + +def test_a_tracked_link_is_not_a_candidate(tmp_path): + """REUSE skips links; only a build input is judged by its target.""" + synth.write(tmp_path / "vendor/x.h", _HEADERS["copyright-and-licence"]) + _link(tmp_path, "src/x.h", "../vendor/x.h") + components, _ = discovery.discover_vendored( - tmp_path, {tmp_path / "src/link/x.h"}, {"NVIDIA"} + tmp_path, set(), {"NVIDIA"}, tracked=["src/x.h", "vendor/x.h"] ) - assert components["vendored:foo-inc"].license_concluded == "BSD-3-Clause" + assert "Files: vendor/x.h." in components["vendored:bar-ltd"].source_info + + +def test_a_link_keeps_its_own_holder_where_its_target_has_no_info(tmp_path): + synth.write(tmp_path / "vendor/x.h", "#pragma once\n") + _link(tmp_path, "src/x.h", "../vendor/x.h") + synth.write(tmp_path / "src/x.h.license", "SPDX-FileCopyrightText: 2020 Foo Inc\n") + + assert list(_discover(tmp_path, "src/x.h")) == ["vendored:foo-inc"] + + +def test_a_link_read_as_its_target_takes_the_targets_precedence(tmp_path): + synth.write(tmp_path / "vendor/x.h", _HEADERS["unparsed-only"]) + synth.write( + tmp_path / "REUSE.toml", + synth.reuse_toml( + synth.annotation("vendor/x.h", "NVIDIA CORPORATION", precedence="aggregate") + ), + ) + _link(tmp_path, "src/x.h", "../vendor/x.h") + + assert list(_discover(tmp_path, "src/x.h")) == ["vendored:bar-ltd"] + + +@pytest.mark.parametrize( + ("name", "head"), + [ + ("icon.svg", "\n\n"), + ("x.c", "/* Copyright 2020 Bar Ltd */\nint x;\n"), + ], + ids=["html", "c"], +) +def test_comment_markup_is_not_part_of_a_scraped_holder(tmp_path, name, head): + component = _vendored(tmp_path, {f"src/{name}": head})["vendored:bar-ltd"] + + assert component.name == "Vendored source (Bar Ltd)" + assert component.copyright_text == "Copyright 2020 Bar Ltd" diff --git a/tests/python/sbom/test_gates.py b/tests/python/sbom/test_gates.py index 299a65cfcb..f1b6578d48 100644 --- a/tests/python/sbom/test_gates.py +++ b/tests/python/sbom/test_gates.py @@ -580,9 +580,14 @@ def test_reuse_toml_declares_the_terms_of_an_untagged_vendored_file(workspace): ) synth.write( workspace.root / "REUSE.toml", - 'version = 1\n\n[[annotations]]\npath = "src/vendor/keys/**"\n' - 'SPDX-FileCopyrightText = "Copyright 2013 The Keys Authors"\n' - "SPDX-" + 'License-Identifier = "BSD-3-Clause"\n', + synth.reuse_toml( + synth.annotation( + "src/vendor/keys/**", + "The Keys Authors", + licence="BSD-3-Clause", + year=2013, + ) + ), ) synth.track(workspace.root) _rebuild(workspace, synth.wheel_payload(workspace)) diff --git a/tests/python/sbom/test_licensing.py b/tests/python/sbom/test_licensing.py index 6b4f028ad3..8bb39308db 100644 --- a/tests/python/sbom/test_licensing.py +++ b/tests/python/sbom/test_licensing.py @@ -389,7 +389,9 @@ def test_a_choice_of_licences_is_not_published_as_both(license_data, tmp_path): "// SPDX-FileCopyrightText: Copyright 2020 Upstream Widgets Ltd\n" "// SPDX-License-Identifier: MIT OR GPL-2.0-or-later\n" ) - components, _ = discovery.discover_vendored(tmp_path, {source}, {"NVIDIA"}) + components, _ = discovery.discover_vendored( + tmp_path, {source}, {"NVIDIA"}, tracked=() + ) component = next(iter(components.values())) assert component.license_concluded == "MIT OR GPL-2.0-or-later" From ff575120777c5079642c6e520633376847357618 Mon Sep 17 00:00:00 2001 From: Andrei Aristarkhov Date: Mon, 19 Oct 2026 19:55:36 -0700 Subject: [PATCH 079/101] build(sbom): inventory static code the build links into a wheel member The collector credited a component only through sources, include paths and DT_NEEDED, so static archives on a link line went unrecorded: _viz.so carries CUDA's libcudart_static.a and no CUDA component said so. It now reads every target's link fragments, resolves -l names the way ld does (link directories, the compiler's, ld's own, linker scripts, -Bstatic), and classifies each input by content. An input is credited to the target, fetched tree, vcpkg port or vendored file that holds it, or to the system package or CUDA toolkit that supplied it; an empty archive is recorded as contributing nothing; anything else stops the Release build. System components take their notices from a DEP-5 copyright file's Copyright fields, and NOASSERTION otherwise. Signed-off-by: Andrei Aristarkhov --- AGENTS.md | 7 +- CMakeLists.txt | 14 +- docs/source/references/sbom.rst | 73 +- scripts/sbom/sbom/build.py | 12 + scripts/sbom/sbom/discovery.py | 919 +++++++++++++++++--- scripts/sbom/sbom/elf.py | 28 +- scripts/sbom/sbom/evidence.py | 32 +- scripts/sbom/sbom/inventory.py | 118 ++- scripts/sbom/sbom/licensing.py | 152 +++- scripts/sbom/sbom/report.py | 1 + tests/python/sbom/synth.py | 229 +++++ tests/python/sbom/test_discovery.py | 5 + tests/python/sbom/test_licensing.py | 102 +++ tests/python/sbom/test_static_archives.py | 982 ++++++++++++++++++++++ 14 files changed, 2484 insertions(+), 190 deletions(-) create mode 100644 tests/python/sbom/test_static_archives.py diff --git a/AGENTS.md b/AGENTS.md index 5369140078..dc92cf1fb9 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -276,10 +276,13 @@ build itself. There is **no checked-in dependency registry** and adding one woul be a second source of truth — declare a dependency the way you already would and it appears in the next wheel's inventory. -Three things fail the Release build rather than shipping a gap, and none is -fixed by excluding a file or writing down an assumption: +These fail the Release build rather than shipping a gap, and none is fixed by +excluding a file or writing down an assumption: - **a wheel member the build cannot explain** — teach the collector that route; +- **static code on a link line the collector cannot trace** — from the build + machine, or installed into the build tree by ExternalProject; link a + traceable copy, or teach the collector that route; - **a redistributed component with no license text** — obtain the terms; - **a checked-in third-party file that fails the per-file REUSE gate** — record upstream's terms in its REUSE info (sbom.rst says how). diff --git a/CMakeLists.txt b/CMakeLists.txt index a802d50e44..e666d0be98 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -3,15 +3,13 @@ cmake_minimum_required(VERSION 3.24...3.25) -# The SBOM collector reads this build's link and compile graph from the file API -# reply, and `cmake_file_api()` is what lets the query live here rather than in -# whatever invoked cmake. It arrived in 3.27, and is feature-tested rather than -# required: raising the project minimum opts into every policy up to it, and -# CMP0148 -- which removes the FindPythonInterp module pybind11's legacy path -# still calls through vcpkg's find_package wrapper -- is one of them. Where the -# command is absent no codemodel is written, and the collector says so itself. +# The SBOM collector reads this build's link and compile graph, and the +# directories the compiler hands the linker, from the file API reply. The query +# is feature-tested rather than required: cmake_file_api() arrived in 3.27, and +# raising the minimum opts into CMP0148, which removes the FindPythonInterp +# module pybind11's legacy path still calls through vcpkg's find_package wrapper. if(COMMAND cmake_file_api) - cmake_file_api(QUERY API_VERSION 1 CODEMODEL 2) + cmake_file_api(QUERY API_VERSION 1 CODEMODEL 2 TOOLCHAINS 1) endif() # Workaround for older CMake policy compatibility in dependencies diff --git a/docs/source/references/sbom.rst b/docs/source/references/sbom.rst index e9f3e258b8..2b7b8fde1a 100644 --- a/docs/source/references/sbom.rst +++ b/docs/source/references/sbom.rst @@ -146,11 +146,16 @@ produced the wheel, so a dependency that builds is a dependency that appears: FetchContent checkouts. ``git`` in each one supplies the commit, the remote, and from the remote the supplier, homepage and purl. +vcpkg's install records + Where the build uses vcpkg, the ``vcpkg/info`` lists under ``VCPKG_INSTALLED_DIR`` (by + default the build tree's ``vcpkg_installed``): which port installed each file. + CMake's file API codemodel The build graph, as CMake itself describes it: which sources were compiled into which - binary, which targets it depends on, and — through include paths — which header-only - dependencies it was compiled against. Followed transitively, so a library linked into a - library linked into the wheel is still reported. The top-level ``CMakeLists.txt`` asks + binary, which targets it depends on, what its link line names, and — + through include paths — which header-only dependencies it was compiled against. + Followed transitively, so a library linked into a library linked into the wheel is still + reported. The top-level ``CMakeLists.txt`` asks for it with ``cmake_file_api()``, which CMake 3.27 added — so a wheel with an SBOM needs at least that, though the project builds without it. Reading the generator's own output instead would only work under Makefiles, since Ninja writes no ``link.txt``. @@ -287,7 +292,61 @@ license from the supplying package's copyright file, or from a license file besi library for toolchains that install outside the package manager. Resolving the SONAME is what licenses the claim — a library the machine cannot find is reported as unexplained, not attributed to something nobody located. No repair tool's directory name is consulted, -so a library the build produced is attributed to the build wherever it ends up. +so a library the build produced is attributed to the build wherever it ends up. Its +copyright notice is read only from the ``Copyright:`` fields of a machine-readable Debian +copyright file, leaving out stanzas that cover only the Debian packaging; any other +text — an EULA, a free-form copyright file — quotes the notices of what it bundles, so +none is asserted, and the packaged text still carries them all. + +Static code leaves no ``DT_NEEDED`` entry behind, so the link line is where it is read — +every fragment CMake records for it, ``-Wl,`` and ``-Xlinker`` words included, with the +values of options known to take one set aside. A path on it is judged by its content, not +its name: an archive or an object file is static code, a shared library is left to +``DT_NEEDED``, and a GNU ``ld`` script — ``libc.so`` is one, naming ``libc_nonshared.a`` — +is followed to the files and ``-l`` names its ``INPUT`` and ``GROUP`` commands give, a +relative name looked up beside the script first, as ``ld`` does. A +``-l`` name is resolved the way ``ld`` resolves it: through the line's ``-L`` directories, +then the directories the compiler hands the linker — CMake's toolchains reply, or the +compiler's own ``-print-search-dirs`` for a language that reply does not cover — then the +linker's built-in ``SEARCH_DIR`` list from ``ld --verbose``, taking a shared library before +an archive unless ``-Bstatic`` is in force. A leading ``=`` in either names the sysroot — +the target's ``CMAKE_SYSROOT``, else the compiler's — as does an absolute name in a script +inside it. ``--push-state`` and ``--pop-state`` are +honoured, and the driver's ``-static`` governs the whole line. The lines read are the +member's own and those of shared libraries it depends on; an executable or module it +merely waits for, such as a code generator, links nothing into it, and neither does +anything reached only through one. + +What is found is then attributed: + +- an archive or object a target of this build produced is that target's contribution; +- one inside a fetched component's tree is that component's; +- one vcpkg installed belongs to the port whose install record + (``vcpkg/info/__.list``) names it. The port is identified by the + upstream source its own ``share//vcpkg.spdx.json`` says it built — a ``pkg:github`` + purl where its one source is on GitHub, ``pkg:generic`` with no supplier where it is not + or where several sources name no single upstream, since purl has no vcpkg type — at the + upstream version, without vcpkg's ``#`` recipe revision. Its licence is read from + ``share//copyright``; +- one tracked in this repository belongs to the holder its REUSE info names, and is this + project's own where that is this project; +- one from the build machine is asked about the same way as a shared library: the package + that owns it and that package's copyright file, or, for an archive under a CUDA toolkit, + the license file the toolkit ships; +- an archive with no members links no code — glibc 2.34 left ``librt.a``, ``libpthread.a`` + and ``libdl.a`` as such stubs — and is recorded in the build evidence rather than as a + component. + +Each component found this way is recorded with ``STATIC_LINK``. Anything else stops the +build: a name that resolves to nothing, a fragment that does not parse, a path that is none +of those kinds — which is also what an unknown option's value looks like — and an archive +inside the build tree that no target produced and no install record names, which is where +ExternalProject leaves its output. Not inventoried: what the compiler driver adds without +naming it on the link line, ``libgcc`` and the C runtime's start files among it; and a +vcpkg port's headers read through an include directory other ports also install into. +Every port shares one include root, so an include directory credits a port only where each +file vcpkg installed under it is that port's; headers read from the shared root are +credited only through a linked archive of the same port. Running the collector locally ----------------------------- @@ -322,7 +381,7 @@ Adding a dependency takes no SBOM change: declare it the way you already would ``FetchContent_Declare``, a tarball under ``deps/`` — and it appears in the next wheel's inventory with its identity, its license and the route it took to get there. -Two things do need attention: +These do need attention: - **A component that states its license nowhere stops the build.** The fix is to obtain the terms, not to record an assumption. For a file checked in here, obtain the terms, then @@ -332,6 +391,10 @@ Two things do need attention: - **A new member that the build cannot explain stops the build.** That means something reached the wheel by a route the collector cannot see; teach it that route rather than excluding the file. +- **Static code on a link line that the collector cannot trace stops the build.** Link + what it can trace — a target of this build, a fetched dependency, a system package or a + known toolkit — or teach it the route that input took, such as an ExternalProject install + into the build tree. Moving to a newer SPDX license list is a dependency bump like any other: change ``SPDX_LICENSE_DATA_COMMIT`` in :code-file:`deps/third_party/CMakeLists.txt`, with its release diff --git a/scripts/sbom/sbom/build.py b/scripts/sbom/sbom/build.py index 74c95c7b4e..a5df0ef11e 100644 --- a/scripts/sbom/sbom/build.py +++ b/scripts/sbom/sbom/build.py @@ -319,6 +319,17 @@ def build( + "\nThey reached the wheel by a route the collector cannot see." ) + if inventory.untraced: + raise BuildError( + "the link lines of wheel members name these, and nothing says whose " + "code they hold:\n " + + "\n ".join(inventory.untraced) + + "\nLink only what the collector can trace -- a target of this build, a " + "fetched dependency's tree, a system package or a known toolkit -- or " + "teach it the route this input took. ExternalProject installs into the " + "build tree by a route it does not know." + ) + unlicensed = sorted( f"{path} ({key}): {reason}" for key, component in inventory.components_present.items() @@ -462,6 +473,7 @@ def build( ) evidence_doc["ci"] = environment_summary() evidence_doc["system_libraries"] = resolver.system_libraries + evidence_doc["static_inputs"] = resolver.static_inputs spdx = document_module.build_document( _project_facts(repo_root, wheel_metadata), diff --git a/scripts/sbom/sbom/discovery.py b/scripts/sbom/sbom/discovery.py index 011e3536e3..349fdf03d8 100644 --- a/scripts/sbom/sbom/discovery.py +++ b/scripts/sbom/sbom/discovery.py @@ -24,13 +24,13 @@ import os import platform import re -import shutil +import shlex import subprocess import tomllib import tarfile import zipfile from collections.abc import Iterable -from dataclasses import dataclass, replace +from dataclasses import dataclass, field, replace from pathlib import Path from packageurl import PackageURL @@ -114,7 +114,8 @@ class Component: key: str name: str - kind: str # "source-tree" | "archive" | "system-library" + # "source-tree" | "archive" | "system-library" | "vendored-source" | "vcpkg-port" + kind: str supplier: str homepage: str purl: str @@ -149,21 +150,7 @@ def repo_head(directory: Path) -> str | None: def git(repo: Path, *args: str) -> str | None: """Run a read-only git query, or return None if git cannot answer.""" - try: - result = subprocess.run( - ["git", "-C", str(repo), *args], - capture_output=True, - text=True, - check=False, - timeout=60, - ) - except (OSError, subprocess.SubprocessError): - return None - return ( - result.stdout.strip() - if result.returncode == 0 and result.stdout.strip() - else None - ) + return _ask("git", "-C", str(repo), *args).strip() or None def notice_from(evidence) -> str: @@ -303,6 +290,225 @@ class _Node: sources: tuple[Path, ...] includes: tuple[Path, ...] dependencies: tuple[str, ...] + # What its link line names. Only linkable targets have one; a static + # library's own requirements land on its consumers' lines. + linked: tuple[Path, ...] = () # outputs of targets in this graph + inputs: tuple[LinkInput, ...] = () # everything else that can carry code + + +@dataclass(frozen=True) +class LinkInput: + """An archive or object a link line names, or a name it could not resolve.""" + + named: str # as the link line spells it + path: Path | None + kind: str # "archive" | "object" | "unresolved" + reason: str = "" + members: int | None = None # an archive's object files + + @property + def key(self) -> str: + return str(self.path) if self.path else self.named + + +@dataclass +class _Line: + """One target's link line while it is read.""" + + link_dir: Path + directories: list[Path] + sysroot: str = "" + linked: list[Path] = field(default_factory=list) + inputs: list[LinkInput] = field(default_factory=list) + + +@dataclass(frozen=True) +class _Search: + """Where `-l` looks after the `-L` directories, and the sysroot `=` names. + + The linker's own directories are read under `sysroot`, which a target's + CMAKE_SYSROOT replaces. + """ + + compiler_dirs: list[Path] + compiler: str | None + sysroot: str + + @property + def directories(self) -> list[Path]: + linker = ( + _linker_search_dirs(self.compiler, self.sysroot) if self.compiler else [] + ) + return self.compiler_dirs + linker + + +# Driver and linker options whose value is the next word. Skipping the value +# keeps a version script or an rpath from being read as an input. +_DRIVER_TAKES_VALUE = frozenset( + """ + -o -x -T -u -e -z -I -B -isystem -include -imacros -MF -MT -MQ + -Xassembler -Xpreprocessor --param + """.split() +) +_LINKER_TAKES_VALUE = frozenset( + """ + -rpath --rpath -rpath-link --rpath-link --version-script -soname --soname -h + -z -T --script -Map --Map --dynamic-list -e --entry -u --undefined --defsym + -o --output --sysroot -y --trace-symbol --wrap -m -R --just-symbols + -dynamic-linker --dynamic-linker -I -F -f --filter --auxiliary --exclude-libs + -plugin --plugin -plugin-opt --plugin-opt --section-start -Ttext -Tdata -Tbss + --retain-symbols-file -Y -G --gpsize -a -A --architecture -b --format -assert + -init -fini -c --mri-script -P --depaudit --audit --require-defined + --export-dynamic-symbol --export-dynamic-symbol-list --dependency-file + --sort-section --section-ordering-file + """.split() +) +_STATIC_MODE = frozenset({"-Bstatic", "-dn", "-non_shared", "-static"}) +_DYNAMIC_MODE = frozenset({"-Bdynamic", "-dy", "-call_shared"}) +# GNU ld script commands that name inputs; AS_NEEDED nests inside them. +_SCRIPT_INPUTS = re.compile(r"\b(?:INPUT|GROUP)\s*\(") +_SCRIPT_COMMENT = re.compile(r"/\*.*?\*/", re.DOTALL) +_SCRIPT_DEPTH = 8 +_NOT_FOUND = ( + "not found; if it is the value of a linker option, that option is one the " + "collector does not know takes a value" +) +_UNKNOWN_KIND = "not an archive, an object, a shared library or a linker script" +_UNSEARCHED = ( + "found in no -L directory, compiler link directory or linker search directory" +) +_NO_COMPILER = ( + "found in no -L directory, and the build records no compiler to ask for its " + "search directories; reconfigure so the file API writes its toolchains reply" +) + + +def _linker_words(fragments: list[dict]) -> tuple[list[str], list[str]]: + """What a link line hands the linker, in order, and fragments that do not parse. + + Every role is read, because CMake files a path given as a link option + under `flags`. Driver options go with their values; `-Wl,` lists and + `-Xlinker` words are unpacked, which is how `--whole-archive` wrappers + arrive. The driver's `-static` governs the whole line wherever it sits. + """ + stream: list[str] = [] + unparsed: list[str] = [] + static = False + for fragment in fragments: + try: + words = iter(shlex.split(fragment["fragment"])) + except ValueError: + unparsed.append(fragment["fragment"]) + continue + if fragment.get("role") == "libraryPath": + stream.extend(f"-L{word.removeprefix('-L')}" for word in words) + continue + for word in words: + if word.startswith("-Wl,"): + stream.extend(word[4:].split(",")) + elif word == "-Xlinker": + stream.append(next(words, "")) + elif word in _DRIVER_TAKES_VALUE: + next(words, None) + elif word == "-static": + static = True + elif word.startswith(("-l", "-L")) or not word.startswith("-"): + stream.append(word) + return (["-Bstatic"] if static else []) + _joined(stream), unparsed + + +def _joined(stream: list[str]) -> list[str]: + """Each search option as one word: `-l x`, `--library x` and `--library=x` + all become `-lx`, and the same for `-L`.""" + words: list[str] = [] + pending = iter(word for word in stream if word) + for word in pending: + option, equals, value = word.partition("=") + if word in ("-L", "--library-path", "-l", "--library"): + option, value = word, next(pending, "") + elif not (equals and option in ("--library-path", "--library")): + words.append(word) + continue + words.append(("-L" if option in ("-L", "--library-path") else "-l") + value) + return words + + +def _search_library(name: str, directories: list[Path], static: bool) -> Path | None: + """The file ld takes for `-l`: `.so` before `.a` in each directory.""" + if name.startswith(":"): + candidates = [name[1:]] + else: + candidates = ([] if static else [f"lib{name}.so"]) + [f"lib{name}.a"] + for directory in directories: + for candidate in candidates: + if (directory / candidate).is_file(): + return directory / candidate + return None + + +def _sysrooted(text: str, sysroot: str, base: Path) -> Path: + """A search directory as ld reads it: a leading `=` stands for the sysroot.""" + if text.startswith("="): + return Path(sysroot + text[1:]).resolve() + return (base / text).resolve() + + +def _inspect(path: Path) -> tuple[str | None, int | None, list[str] | None]: + """(kind, archive members, script entries), judged by content. + + The kind is "archive", "object", "shared", "script", "missing", or None for + a file that is none of them. + """ + if not path.is_file(): + return "missing", None, None + members = archive_members(path) + if members is not None: + return "archive", members, None + kind = {"ET_REL": "object", "ET_DYN": "shared"}.get(elf.file_type(path) or "") + if kind: + return kind, None, None + entries = _script_inputs(path) + return ("script" if entries is not None else None), None, entries + + +def _script_inputs(path: Path) -> list[str] | None: + """The files and `-l` names a GNU ld script's INPUT and GROUP commands name. + + None if the file is not such a script. + """ + try: + with path.open("rb") as handle: + head = handle.read(1 << 16) + except OSError: + return None + if b"\0" in head: + return None + body = _SCRIPT_COMMENT.sub(" ", head.decode("utf-8", "replace")) + commands = list(_SCRIPT_INPUTS.finditer(body)) + if not commands: + return None + found: list[str] = [] + for match in commands: + depth, end = 1, match.end() + while end < len(body) and depth: + depth += {"(": 1, ")": -1}.get(body[end], 0) + end += 1 + found += [ + item + for item in re.split(r"[\s,()]+", body[match.end() : end - 1]) + if item and item != "AS_NEEDED" + ] + return found + + +def owned_input(item: LinkInput, ownership: Ownership | None) -> str | None: + """The component a linked archive or object belongs to by the ownership + index -- a fetched tree, a vcpkg port, vendored material -- if it holds code.""" + if ownership is None or item.path is None or item.kind == "unresolved": + return None + if item.members == 0: + return None + return ownership.owner_of(item.path) def _owner(path: Path, ownership: Ownership | None) -> str | None: @@ -341,8 +547,8 @@ def check(path: Path) -> bool: return check -def _codemodel_file(reply: Path) -> Path | None: - """The current reply's codemodel, reached through its index. +def _reply_objects(reply: Path) -> tuple[dict[str, Path], str]: + """The current reply's objects by kind, reached through its index, and the generator. Every other reply filename is documented as unspecified and not to be interpreted, and CMake only *attempts* to delete what it did not just @@ -351,21 +557,118 @@ def _codemodel_file(reply: Path) -> Path | None: order: "the one with the largest name in lexicographic order is the current index file". """ - if not reply.is_dir(): - return None - indexes = sorted(reply.glob("index-*.json")) + indexes = sorted(reply.glob("index-*.json")) if reply.is_dir() else [] if not indexes: - return None + return {}, "" index = json.loads(indexes[-1].read_text(encoding="utf-8")) + generator = index.get("cmake", {}).get("generator", {}).get("name", "") + wanted = {"codemodel": 2, "toolchains": 1} + objects: dict[str, Path] = {} for item in index.get("objects", []): version = item.get("version") or {} - if item.get("kind") == "codemodel" and version.get("major") == 2: + if item.get("kind") in wanted and wanted[item["kind"]] == version.get("major"): # Index references are relative to the reply directory. found = reply / item["jsonFile"] - return found if found.is_file() else None + if found.is_file(): + objects[item["kind"]] = found + return objects, generator + + +_CACHE_COMPILER = re.compile(r"^CMAKE_(?P[A-Z]+)_COMPILER$") +_SEARCH_DIR = re.compile(r'SEARCH_DIR\("(?P[^"]+)"\)') + + +def cmake_cache(build_dir: Path) -> dict[str, str]: + """Every entry of a build tree's CMakeCache.txt, by name; empty if it has none.""" + cache = build_dir / "CMakeCache.txt" + if not cache.is_file(): + return {} + entries: dict[str, str] = {} + for line in cache.read_text(encoding="utf-8", errors="replace").splitlines(): + if line.startswith(("#", "//")) or "=" not in line or ":" not in line: + continue + entries[line.split(":", 1)[0]] = line.split("=", 1)[1] + return entries + + +def _search_directories(toolchains: Path | None, build_dir: Path) -> dict[str, _Search]: + """Where `-l` looks after the `-L` directories, by link language. + + First the directories the compiler hands the linker: CMake's toolchains + reply, or the compiler asked directly for a language the reply does not + cover -- CMake records them only for languages the top-level project + enables. Then the linker's own built-in SEARCH_DIRs. + """ + compilers = { + match["language"]: value + for name, value in cmake_cache(build_dir).items() + if (match := _CACHE_COMPILER.match(name)) + } + recorded: dict[str, list[str] | None] = {} + if toolchains is not None: + payload = json.loads(toolchains.read_text(encoding="utf-8")) + for item in payload.get("toolchains", []): + compiler = item.get("compiler", {}) + recorded[item["language"]] = compiler.get("implicit", {}).get( + "linkDirectories" + ) + if compiler.get("path"): + compilers[item["language"]] = compiler["path"] + + found: dict[str, _Search] = {} + for language in sorted(set(compilers) | set(recorded)): + compiler = compilers.get(language) + listed = recorded.get(language) + if listed is not None: + directories = [Path(item) for item in listed] + elif compiler: + directories = _compiler_search_dirs(compiler) + else: + directories = None + if directories is None: + continue + sysroot = _ask(compiler, "-print-sysroot").strip() if compiler else "" + found[language] = _Search(directories, compiler, sysroot) + return found + + +def _ask(*command: str) -> str: + """A command's standard output, or empty where it cannot run or fails.""" + try: + result = subprocess.run( + list(command), capture_output=True, text=True, check=False, timeout=60 + ) + except (OSError, subprocess.SubprocessError): + return "" + return result.stdout if result.returncode == 0 else "" + + +@functools.lru_cache(maxsize=None) +def _compiler_search_dirs(compiler: str) -> list[Path] | None: + """The `libraries:` list of ` -print-search-dirs`, GCC's and Clang's.""" + for line in _ask(compiler, "-print-search-dirs").splitlines(): + if line.startswith("libraries:"): + listed = line.split(":", 1)[1].strip().removeprefix("=") + return [ + Path(item).resolve() + for item in listed.split(":") + if Path(item).is_dir() + ] return None +@functools.lru_cache(maxsize=None) +def _linker_search_dirs(compiler: str, sysroot: str) -> list[Path]: + """The SEARCH_DIRs of the linker the compiler runs, from `ld --verbose`.""" + linker = _ask(compiler, "-print-prog-name=ld").strip() or "ld" + found = [] + for match in _SEARCH_DIR.finditer(_ask(linker, "--verbose")): + directory = _sysrooted(match["path"], sysroot, Path("/")) + if directory.is_dir(): + found.append(directory) + return found + + class BuildGraph: """The link and compile graph, read from CMake's own file API. @@ -382,16 +685,26 @@ def __init__(self, build_dir: Path, config: str | None = None) -> None: self._nodes: dict[str, _Node] = {} self._by_name: dict[str, list[Artifact]] = {} self._closures: dict[str, tuple[_Node, ...]] = {} + self._static: dict[str, dict[LinkInput, set[str]]] = {} + self._inspected: dict[ + Path, tuple[str | None, int | None, list[str] | None] + ] = {} self._load(config) def _load(self, config: str | None) -> None: reply = self.build_dir / ".cmake" / "api" / "v1" / "reply" - model_file = _codemodel_file(reply) - if model_file is None: + objects, generator = _reply_objects(reply) + if "codemodel" not in objects: raise FileApiError( f"no CMake file API codemodel under {reply}. Configure the project " "with CMake 3.27 or newer, which is where cmake_file_api() asks for it." ) + model_file = objects["codemodel"] + search = _search_directories(objects.get("toolchains"), self.build_dir) + # A relative link-line path is relative to where the link runs: the + # target's own build directory under the Makefile generators, the top + # of the build tree under Ninja. + per_directory = "Makefiles" in generator or "WMake" in generator codemodel = json.loads(model_file.read_text(encoding="utf-8")) source_root = Path(codemodel["paths"]["source"]) @@ -417,6 +730,7 @@ def resolve(raw: str, root: Path) -> Path: path = Path(raw) return path if path.is_absolute() else (root / path) + lines: dict[str, tuple[list[str], list[str], Path, _Search | None]] = {} for entry in chosen["targets"]: # Relative to the codemodel, which is what the format says, even # though CMake writes it into the reply directory alongside these. @@ -428,6 +742,20 @@ def resolve(raw: str, root: Path) -> Path: for group in target.get("compileGroups", []) for item in group.get("includes", []) } + link_dir = ( + resolve(target.get("paths", {}).get("build", "."), build_root) + if per_directory + else build_root + ) + link = target.get("link", {}) + words, unparsed = _linker_words(link.get("commandFragments", [])) + found = search.get(link.get("language", "")) + # CMAKE_SYSROOT reaches the linker as --sysroot, which the compiler's + # own -print-sysroot does not report. + sysroot = link.get("sysroot", {}).get("path") + if found is not None and sysroot: + found = replace(found, sysroot=sysroot) + lines[entry["id"]] = (words, unparsed, link_dir, found) self._nodes[entry["id"]] = _Node( name=target["name"], kind=target["type"], @@ -448,6 +776,119 @@ def resolve(raw: str, root: Path) -> Path: self.artifacts[output] = artifact self._by_name.setdefault(output.name, []).append(artifact) + # Read once every target's outputs are known: an output of this graph + # is that target's contribution, whatever its content. + for target_id, (words, unparsed, link_dir, found) in lines.items(): + line = self._read_line(words, link_dir, found) + line.inputs += [ + LinkInput(item, None, "unresolved", "does not parse as shell words") + for item in unparsed + ] + self._nodes[target_id] = replace( + self._nodes[target_id], + linked=tuple(dict.fromkeys(line.linked)), + inputs=tuple(dict.fromkeys(line.inputs)), + ) + + def _read_line( + self, words: list[str], link_dir: Path, search: _Search | None + ) -> _Line: + """What a linker word stream names. + + ld applies every `-L` to every `-l`, then the compiler's directories, + then its own built-in ones -- `search`, or None where the build records + no compiler. A `-l` name that ld would satisfy with a shared library + adds nothing, since DT_NEEDED records it. + """ + sysroot = search.sysroot if search else "" + line = _Line( + link_dir, + [ + _sysrooted(word[2:], sysroot, link_dir) + for word in words + if word.startswith("-L") + ] + + (search.directories if search else []), + sysroot, + ) + static = False + saved: list[bool] = [] + pending = iter(words) + for word in pending: + if word in _STATIC_MODE: + static = True + elif word in _DYNAMIC_MODE: + static = False + elif word == "--push-state": + saved.append(static) + elif word == "--pop-state": + static = saved.pop() if saved else False + elif word.startswith("-l"): + hit = _search_library(word[2:], line.directories, static) + if hit is None: + reason = _UNSEARCHED if search else _NO_COMPILER + line.inputs.append(LinkInput(word, None, "unresolved", reason)) + else: + self._classify(word, hit, line, static) + elif word.startswith("-"): + if word in _LINKER_TAKES_VALUE: + next(pending, None) + else: + self._classify(word, link_dir / word, line, static) + return line + + def _classify( + self, spelling: str, path: Path, line: _Line, static: bool, depth: int = 0 + ) -> None: + """Add what one path on a link line brings in, a linker script followed + under the `-Bstatic` state in force where the line names it. + + A shared library adds nothing, since DT_NEEDED records it. + """ + # ld looks beside the name it was given, not beside a symlink's target. + beside = path.parent + path = path.resolve() + if path in self.artifacts: + line.linked.append(path) + return + if path not in self._inspected: + self._inspected[path] = _inspect(path) + kind, members, entries = self._inspected[path] + if kind in ("archive", "object"): + line.inputs.append(LinkInput(spelling, path, kind, members=members)) + elif kind == "missing": + line.inputs.append(LinkInput(spelling, path, "unresolved", _NOT_FOUND)) + elif kind is None: + line.inputs.append(LinkInput(spelling, path, "unresolved", _UNKNOWN_KIND)) + elif kind == "script" and depth > _SCRIPT_DEPTH: + line.inputs.append( + LinkInput( + spelling, path, "unresolved", "linker scripts nest too deeply here" + ) + ) + elif kind == "script": + # An absolute name in a script inside the sysroot, or one opening + # with `=`, is read under the sysroot. + in_sysroot = bool(line.sysroot) and path.is_relative_to( + Path(line.sysroot).resolve() + ) + for entry in entries or []: + if entry.startswith("-l"): + hit = _search_library(entry[2:], line.directories, static) + else: + if entry.startswith("=") or (in_sysroot and entry.startswith("/")): + entry = line.sysroot + entry.removeprefix("=") + # Beside the script, then where the link runs, then the + # search path; an absolute name is itself in each. + hit = _search_library( + ":" + entry, [beside, line.link_dir, *line.directories], static + ) + if hit is None: + named = f"{spelling} ({path.name}: {entry})" + line.inputs.append(LinkInput(named, None, "unresolved", _NOT_FOUND)) + else: + self._classify(spelling, hit, line, static, depth + 1) + def by_name(self, name: str) -> list[Artifact]: return self._by_name.get(name, []) @@ -474,15 +915,14 @@ def repo_inputs(self, repo_root: Path, build_dir: Path) -> set[Path]: ) return found - def _closure(self, target_id: str) -> tuple[_Node, ...]: - """The target and everything it depends on, transitively. + def _reach(self, target_id: str, prune=None) -> tuple[_Node, ...]: + """The target and everything it depends on or links, transitively. - Memoised per instance, not with `lru_cache`: a cache on the class would - hold every graph ever built alive for the life of the process. + A link line naming another target's output brings that target, whether + or not it is also a declared dependency. A node `prune` accepts, other + than the target itself, is left out with everything reached only + through it. """ - cached = self._closures.get(target_id) - if cached is not None: - return cached seen: set[str] = set() stack = [target_id] found: list[_Node] = [] @@ -492,11 +932,38 @@ def _closure(self, target_id: str) -> tuple[_Node, ...]: continue seen.add(current) node = self._nodes[current] + if prune is not None and current != target_id and prune(node): + continue found.append(node) stack.extend(node.dependencies) - self._closures[target_id] = tuple(found) + stack.extend(self.artifacts[item].target_id for item in node.linked) + return tuple(found) + + def _closure(self, target_id: str) -> tuple[_Node, ...]: + """`_reach`, memoised per instance, not with `lru_cache`: a cache on the + class would hold every graph ever built alive for the life of the process.""" + if target_id not in self._closures: + self._closures[target_id] = self._reach(target_id) return self._closures[target_id] + def static_inputs(self, artifact: Artifact) -> dict[LinkInput, set[str]]: + """What the closure links that no target here produced. + + Maps each to the targets whose link line names it. Another executable + or module is a build-order dependency -- a code generator, say -- and + nothing links it or what it links. + """ + if artifact.target_id not in self._static: + found: dict[LinkInput, set[str]] = {} + for node in self._reach( + artifact.target_id, + lambda node: node.kind in ("EXECUTABLE", "MODULE_LIBRARY"), + ): + for item in node.inputs: + found.setdefault(item, set()).add(node.name) + self._static[artifact.target_id] = found + return self._static[artifact.target_id] + def contributions( self, artifact: Artifact, ownership: Ownership | None = None ) -> dict[str, set[str]]: @@ -504,7 +971,7 @@ def contributions( Walks the target's dependency closure. Compiled sources name a component outright; include paths are the only trace a header-only dependency - leaves anywhere. + leaves anywhere; a linked archive or object names whoever holds it. """ found: dict[str, set[str]] = {} @@ -526,6 +993,10 @@ def note(key: str, how: str) -> None: # way a fetched include directory is. for owner in ownership.owners_under(include): note(owner, "header-include") + for item in self.static_inputs(artifact): + owner = owned_input(item, ownership) + if owner: + note(owner, "prebuilt-library") return found def first_party( @@ -559,20 +1030,30 @@ class Ownership: """ def __init__(self) -> None: - self._files: dict[Path, str] = {} + # path -> (owner, whether an include directory credits it outright) + self._files: dict[Path, tuple[str, bool]] = {} self._roots: list[tuple[Path, str]] = [] + # directory -> (owners credited outright, owners credited if alone) + self._index: dict[Path, tuple[set[str], set[str]]] | None = None def register_root(self, root: Path, key: str) -> None: self._roots.append((root.resolve(), key)) - def register_file(self, path: Path, key: str) -> None: - self._files[path.resolve()] = key + def register_file(self, path: Path, key: str, *, by_directory: bool = True) -> None: + """Record a file's owner. + + An include directory credits every owner of a file under it; one whose + files are registered with `by_directory=False` only where it holds every + such file there. + """ + self._files[path.resolve()] = (key, by_directory) + self._index = None def owner_of(self, path: Path) -> str | None: resolved = path.resolve() - owner = self._files.get(resolved) - if owner is not None: - return owner + entry = self._files.get(resolved) + if entry is not None: + return entry[0] # Longest root wins, so a vendored subdirectory beats its parent. best: tuple[int, str] | None = None for root, key in self._roots: @@ -584,19 +1065,29 @@ def owner_of(self, path: Path) -> str | None: def owners_under(self, directory: Path) -> set[str]: """Components with files inside a directory, for include-path evidence.""" - prefix = str(directory.resolve()) - found = { - key - for path, key in self._files.items() - if str(path) == prefix or str(path).startswith(prefix + "/") - } + resolved = directory.resolve() + outright, alone = self._by_directory().get(resolved, (set(), set())) + found = set(outright) + prefix = str(resolved) found.update( key for root, key in self._roots if str(root) == prefix or str(root).startswith(prefix + "/") ) + if len(alone) == 1: + found |= alone return found + def _by_directory(self) -> dict[Path, tuple[set[str], set[str]]]: + """Each registered file's owner under the file and every directory above it.""" + if self._index is None: + self._index = {} + for path, (key, outright) in self._files.items(): + for directory in (path, *path.parents): + entry = self._index.setdefault(directory, (set(), set())) + entry[0 if outright else 1].add(key) + return self._index + # ============================================================================== # Third-party sources vendored in this repository @@ -929,11 +1420,123 @@ def discover_vendored( return components, grouped +# ============================================================================== +# vcpkg +# ============================================================================== +# vcpkg installs into the build tree, so what it builds is neither a target of +# this build nor a fetched checkout. Its install records name the port that +# put each file there, as dpkg's do on the build host. + + +def discover_vcpkg( + installed: Path, target_triplet: str | None = None +) -> tuple[dict[str, Component], dict[str, list[Path]]]: + """Every port vcpkg installed under `installed`, and the files each owns. + + A port installed for the host as well is described from its install for + `target_triplet` -- what the build links -- where there is one. + """ + info = installed / "vcpkg" / "info" + components: dict[str, Component] = {} + owned: dict[str, list[Path]] = {} + for listing in sorted(info.glob("*.list")) if info.is_dir() else []: + # __.list; port names carry no underscore. + port, _, rest = listing.stem.partition("_") + version, _, triplet = rest.rpartition("_") + key = f"vcpkg:{port}" + # A port installed for both the host and the target triplet has a list + # for each, and both installs are its files. + owned.setdefault(key, []).extend( + installed / line + for line in listing.read_text(encoding="utf-8").splitlines() + if line and not line.endswith("/") + ) + if key not in components or triplet == target_triplet: + components[key] = _vcpkg_component(key, port, version, triplet, installed) + return components, owned + + +def _vcpkg_component( + key: str, port: str, version: str, triplet: str, installed: Path +) -> Component: + """A port, identified by the upstream source its own SBOM says it built. + + vcpkg writes `share//vcpkg.spdx.json` with the recipe and the + downloaded sources, and copies upstream's licence to `share//copyright`. + """ + share = installed / triplet / "share" / port + try: + sbom = json.loads((share / "vcpkg.spdx.json").read_text(encoding="utf-8")) + except (OSError, ValueError): + sbom = {} + packages = {item.get("SPDXID"): item for item in sbom.get("packages", [])} + recipe = packages.get("SPDXRef-port", {}) + sources = [ + item["downloadLocation"] + for spdx_id, item in packages.items() + if str(spdx_id).startswith("SPDXRef-resource-") and item.get("downloadLocation") + ] + port_version = recipe.get("versionInfo") or version + # `#` is the recipe's revision, not anything upstream released. + version = port_version.partition("#")[0] + # One source names the upstream; several (or none) name no single one. + upstream = sources[0] if len(sources) == 1 else None + location = (upstream or "").removeprefix("git+") + remote, _, ref = ( + location.rpartition("@") + if "@" in location.rsplit("/", 1)[-1] + else (location, "", "") + ) + if remote and _GITHUB_REMOTE.search(remote): + supplier, homepage, purl, download = _identity(remote, port, ref or None) + else: + supplier, homepage, download = "NOASSERTION", "NOASSERTION", upstream + purl = PackageURL(type="generic", name=port, version=version).to_string() + homepage = recipe.get("homepage") or homepage + + evidence = [] + licence = share / "copyright" + if licence.is_file(): + evidence.append( + licensing.evidence_for( + key, + "vcpkg-port", + licence.relative_to(installed.parent).as_posix(), + licence.read_text(encoding="utf-8", errors="replace"), + "grant", + ) + ) + concluded, declared = licensing.expression(evidence, key) + details = [ + f"Built and installed by vcpkg port {port} {port_version} for {triplet}." + ] + if upstream: + details.append(f"Source: {upstream}.") + if recipe.get("downloadLocation"): + details.append(f"Port recipe: {recipe['downloadLocation']}.") + return Component( + key=key, + name=port, + kind="vcpkg-port", + supplier=supplier, + homepage=homepage, + purl=purl, + version=version, + download_location=download or "NOASSERTION", + source_info=" ".join(details), + license_concluded=concluded, + license_declared=declared, + copyright_text=notice_from(evidence), + evidence=tuple(evidence), + ) + + # ============================================================================== # The build host # ============================================================================== -# The last discovery source: a shared library that no index explains was not -# produced or fetched by this build, so ask the machine that supplied it. +# The last discovery source: a shared library that no index explains, or a +# static archive linked from outside the tree, was not produced or fetched by +# this build, so ask the machine that supplied it. _SEARCH_ROOTS = ( "/usr/lib/x86_64-linux-gnu", @@ -948,24 +1551,11 @@ def discover_vendored( def _host_library_paths() -> list[Path]: """Every shared library the loader can see, plus the CUDA trees it cannot.""" paths: set[Path] = set() - if shutil.which("ldconfig"): - try: - result = subprocess.run( - ["ldconfig", "-p"], - capture_output=True, - text=True, - check=False, - timeout=60, - ) - for line in result.stdout.splitlines(): - _, _, target = line.partition("=>") - target = target.strip() - if target: - paths.add(Path(target)) - except (OSError, subprocess.SubprocessError): - pass - roots = _library_roots() - for root in roots: + for line in _ask("ldconfig", "-p").splitlines(): + _, _, target = line.partition("=>") + if target.strip(): + paths.add(Path(target.strip())) + for root in _library_roots(): paths.update(item for item in root.glob("*.so*") if item.is_file()) return sorted(paths) @@ -993,60 +1583,90 @@ def _host_build_id_index() -> dict[str, str]: return index -def resolve_system_library_by_build_id(build_id: str) -> dict: - """Trace a vendored library back by build-id, whatever it was renamed to.""" - origin: dict = { - "soname": None, - "build_id": build_id, - "resolved_path": None, - "package": None, - "copyright": None, - } - found = _host_build_id_index().get(build_id) - if found is None: +def _host_origin(library: Path | None, **known) -> dict: + """What the build machine says about a file: where it resolves, and the + package that owns it with that package's terms.""" + origin: dict = {**known, "resolved_path": None, "package": None, "copyright": None} + if library is None: return origin - library = Path(found).resolve() - origin["soname"] = library.name + library = library.resolve() origin["resolved_path"] = str(library) return _identify_host_package(library, origin) +def resolve_system_library_by_build_id(build_id: str) -> dict: + """Trace a vendored library back by build-id, whatever it was renamed to.""" + found = _host_build_id_index().get(build_id) + library = Path(found).resolve() if found else None + return _host_origin( + library, soname=library.name if library else None, build_id=build_id + ) + + def resolve_system_library(soname: str) -> dict: """Trace an auditwheel-vendored library back to the package that supplied it.""" - origin: dict = { - "soname": soname, - "resolved_path": None, - "package": None, - "copyright": None, - } - roots = _library_roots() + library = next( + (root / soname for root in _library_roots() if (root / soname).exists()), None + ) + return _host_origin(library, soname=soname) - library = next((root / soname for root in roots if (root / soname).exists()), None) - if library is None: - return origin - library = library.resolve() - origin["resolved_path"] = str(library) - return _identify_host_package(library, origin) + +def resolve_static_archive(archive: Path) -> dict: + """Trace an archive or object the build linked to the package or toolkit it came from. + + A shared library is explained once its SONAME resolves on the machine: that + is how the repair tool found the file it vendored. A link line already + names a static input's path, so finding it proves nothing; only a package + or a toolkit that states its terms explains one. + """ + if not archive.is_file(): + return _host_origin(None, toolkit_root=None) + root = _toolkit_root(archive) + return _host_origin(archive, toolkit_root=str(root) if root else None) + + +def archive_members(path: Path) -> int | None: + """How many object files a `!` archive holds; None if it is not one. + + glibc 2.34 folded librt, libpthread and libdl into libc and left each as an + archive with no members, so naming one on a link line links nothing. + """ + try: + with path.open("rb") as handle: + magic = handle.read(8) + if magic not in (b"!\n", b"!\n"): + return None + count = 0 + while header := handle.read(60): + if len(header) < 60 or header[58:60] != b"`\n": + return None + name = header[:16].rstrip() + size = int(header[48:58]) + skip = size + size % 2 + # BSD writes a long name at the head of the member's data. + if name.startswith(b"#1/"): + length = int(name[3:]) + name = handle.read(length).rstrip(b"\0") + skip -= length + # The symbol table and the long-name table are not members. + index = name in (b"/", b"//", b"/SYM64/") or name.startswith( + b"__.SYMDEF" + ) + count += not index + # A thin archive stores only its index; members stay outside it. + if index or magic == b"!\n": + handle.seek(skip, os.SEEK_CUR) + return count + except (OSError, ValueError): + return None def _identify_host_package(library: Path, origin: dict) -> dict: """Name the distro package that owns a resolved host library.""" - if shutil.which("dpkg-query") is None: - return origin - try: - result = subprocess.run( - ["dpkg-query", "-S", str(library)], - capture_output=True, - text=True, - check=False, - timeout=60, - ) - except (OSError, subprocess.SubprocessError): + owned = _ask("dpkg-query", "-S", str(library)) + if ":" not in owned: return origin - if result.returncode != 0 or ":" not in result.stdout: - return origin - - package = result.stdout.split(":", 1)[0].strip() + package = owned.split(":", 1)[0].strip() origin["package"] = package origin["supplier"] = _package_supplier(package) origin["version"] = _package_field(package, "Version") @@ -1059,18 +1679,7 @@ def _identify_host_package(library: Path, origin: dict) -> dict: def _package_field(package: str, field: str) -> str | None: """One field of a distro package's metadata, as dpkg records it.""" - try: - result = subprocess.run( - ["dpkg-query", "-W", f"-f=${{{field}}}", package], - capture_output=True, - text=True, - check=False, - timeout=60, - ) - except (OSError, subprocess.SubprocessError): - return None - value = result.stdout.strip() - return value if result.returncode == 0 and value else None + return _ask("dpkg-query", "-W", f"-f=${{{field}}}", package).strip() or None def _os_release_id() -> str | None: @@ -1100,17 +1709,39 @@ def _package_supplier(package: str) -> str | None: def _library_roots() -> list[Path]: """Where a pre-built library the build machine supplied can be found.""" roots = [Path(item) for item in _SEARCH_ROOTS if Path(item).is_dir()] - return roots + [Path(item) for item in sorted(glob.glob("/usr/local/cuda*/lib64"))] + return roots + [root / "lib64" for root in _toolkit_roots()] + + +@functools.lru_cache(maxsize=None) +def _toolkit_roots() -> list[Path]: + """Toolkits that may install outside the package manager: CUDA's.""" + return sorted( + { + Path(item).resolve() + for item in glob.glob("/usr/local/cuda*") + if Path(item).is_dir() + } + ) + + +def _toolkit_root(path: Path) -> Path | None: + resolved = path.resolve() + return next( + (root for root in _toolkit_roots() if resolved.is_relative_to(root)), None + ) def _license_near(library: Path) -> Path | None: """A toolchain that installs outside the package manager still ships its terms. - CUDA is the case that matters: the runtime lands in /usr/local/cuda*/lib64 with - EULA.txt a level or two up, and dpkg knows nothing about it. + CUDA is the case that matters: its libraries resolve into + targets//lib with EULA.txt at the toolkit root, and dpkg need know + nothing about it. Under a toolkit root the search runs up to that root. """ - current = library.parent - for _ in range(3): + root = _toolkit_root(library) + current = library.resolve().parent + levels = 3 if root is None else len(current.relative_to(root).parts) + 1 + for _ in range(levels): for entry in sorted(current.iterdir()) if current.is_dir() else []: if entry.is_file() and licensing.classify(entry.name): return entry @@ -1120,10 +1751,30 @@ def _license_near(library: Path) -> Path | None: return None -def system_component(soname: str, origin: dict) -> Component: - """A pre-built library the build machine supplied, named by what it came from.""" +def _system_notice(evidence: list[licensing.LicenseEvidence]) -> str: + """The holders a machine-readable Debian copyright file names, or NOASSERTION. + + Any other text -- an EULA, a free-form copyright file -- quotes the notices + of whatever it bundles, and scraping those names other people's code as the + holders of this library. The packaged text still carries all of it. + """ + for item in evidence: + fields = licensing.dep5_copyright(item.text) + if fields: + return "\n".join(licensing.fold_notices(fields)) + return "NOASSERTION" + + +def system_component( + soname: str, origin: dict, route: str | None = None, key: str | None = None +) -> Component: + """A pre-built library the build machine supplied, named by what it came from. + + `route` says how it reached the wheel; the default is auditwheel's. `key` + defaults to `system:`. + """ evidence: list[licensing.LicenseEvidence] = [] - key = f"system:{soname}" + key = key or f"system:{soname}" license_path = origin.get("copyright") if not license_path and origin.get("resolved_path"): found = _license_near(Path(origin["resolved_path"])) @@ -1141,7 +1792,8 @@ def system_component(soname: str, origin: dict) -> Component: package = origin.get("package") details = [ - f"Vendored into the wheel by auditwheel from {origin.get('resolved_path') or soname}." + route + or f"Vendored into the wheel by auditwheel from {origin.get('resolved_path') or soname}." ] if package: details.append(f"Supplied by system package {package}.") @@ -1150,10 +1802,7 @@ def system_component(soname: str, origin: dict) -> Component: name=soname, kind="system-library", supplier=origin.get("supplier") or "NOASSERTION", - # The distro copyright file is where the holder is stated, and it was - # read, packaged and put on the file record -- then left off the package - # and out of the notices every other component appears in. - copyright_text=notice_from(evidence), + copyright_text=_system_notice(evidence), homepage="NOASSERTION", # A distro package the machine can name resolves; pkg:generic does not. purl=( diff --git a/scripts/sbom/sbom/elf.py b/scripts/sbom/sbom/elf.py index 8769c9bbf9..fa948f6add 100644 --- a/scripts/sbom/sbom/elf.py +++ b/scripts/sbom/sbom/elf.py @@ -71,25 +71,29 @@ def build_id_of(data: bytes) -> str | None: return None -def read_build_id(path: Path) -> str | None: - """The build-id of a file on disk, read without loading it. +def _from_file(path: Path, read): + """`read(ELFFile)` over a file on disk, or None if it is not readable ELF. - Scanning a machine's libraries means touching hundreds of files, some of - them hundreds of megabytes. pyelftools seeks, so only the headers and the - note are ever read. + pyelftools seeks, so only the headers and what `read` asks for are read -- + which matters when scanning hundreds of a machine's libraries, some of + them hundreds of megabytes. Scanning also meets files that only look like + ELF, and a truncated one raises `struct.error` from the parser. """ try: with path.open("rb") as handle: if handle.read(4) != ELF_MAGIC: return None handle.seek(0) - return _build_id_from(ELFFile(handle)) - except (OSError, ELFError): - # Scanning a machine's libraries meets files that only look like ELF. - # Anything else raising here is a bug in this parser, not bad input. + return read(ELFFile(handle)) + except (OSError, ELFError, struct.error): return None +def read_build_id(path: Path) -> str | None: + """The build-id of a file on disk, read without loading it.""" + return _from_file(path, _build_id_from) + + def read_dynamic(data: bytes) -> DynamicInfo: """Parse SONAME, NEEDED and the build-id out of an in-memory ELF image.""" if not is_elf(data): @@ -117,3 +121,9 @@ def read_dynamic(data: bytes) -> DynamicInfo: machine=_ARCH.get(machine, machine), build_id=_build_id_from(elffile), ) + + +def file_type(path: Path) -> str | None: + """A file's ELF `e_type` -- `ET_REL` for an object, `ET_DYN` for a shared + library -- or None if it is not an ELF file.""" + return _from_file(path, lambda elffile: elffile.header["e_type"]) diff --git a/scripts/sbom/sbom/evidence.py b/scripts/sbom/sbom/evidence.py index c24dc77fcd..7f8bdbadfd 100644 --- a/scripts/sbom/sbom/evidence.py +++ b/scripts/sbom/sbom/evidence.py @@ -23,11 +23,13 @@ git, BuildGraph, Component, + cmake_cache, FileIndex, discover_archives, discover_vendored, project_authors, discover_source_trees, + discover_vcpkg, notice_from, tracked_files, EMPTY_SHA256, @@ -76,14 +78,11 @@ def _cmake_cache(build_dir: Path) -> dict[str, str]: raise EvidenceError( f"{cache} not found; point --build-dir at a configured build tree" ) - entries: dict[str, str] = {} - for line in cache.read_text(encoding="utf-8", errors="replace").splitlines(): - if line.startswith(("#", "//")) or "=" not in line or ":" not in line: - continue - name = line.split(":", 1)[0] - if _CACHE_KEYS.match(name): - entries[name] = line.split("=", 1)[1] - return entries + return { + name: value + for name, value in cmake_cache(build_dir).items() + if _CACHE_KEYS.match(name) + } def _fetched_from(archive: Path) -> str | None: @@ -280,6 +279,17 @@ def discover(repo_root: Path, build_dir: Path) -> Discovery: if root.is_dir(): ownership.register_root(root, root.name.removesuffix("-build")) + # What vcpkg built, where VCPKG_INSTALLED_DIR (by default the build tree's + # vcpkg_installed) records which port installed each file. + ports, port_files = discover_vcpkg( + Path(cache.get("VCPKG_INSTALLED_DIR") or build_dir / "vcpkg_installed"), + cache.get("VCPKG_TARGET_TRIPLET"), + ) + components.update(ports) + for key, paths in port_files.items(): + for path in paths: + ownership.register_file(path, key, by_directory=False) + # Third-party material checked in here rather than fetched. Tracked files as # well as build inputs: Python or data reaching the wheel is compiled by nothing. vendored, owned_paths = discover_vendored( @@ -332,7 +342,10 @@ def document( artifacts = [] for artifact in discovery.graph.artifacts.values(): contributions = discovery.graph.contributions(artifact, discovery.ownership) - if not contributions: + # Resolved to components only where a wheel member links them; see + # `static_inputs`. + inputs = sorted({item.key for item in discovery.graph.static_inputs(artifact)}) + if not contributions and not inputs: continue try: output = artifact.output.relative_to(discovery.build_dir).as_posix() @@ -348,6 +361,7 @@ def document( "contributions": { key: sorted(how) for key, how in sorted(contributions.items()) }, + **({"static_inputs": inputs} if inputs else {}), } ) diff --git a/scripts/sbom/sbom/inventory.py b/scripts/sbom/sbom/inventory.py index 360f6ae950..155a84d400 100644 --- a/scripts/sbom/sbom/inventory.py +++ b/scripts/sbom/sbom/inventory.py @@ -12,6 +12,7 @@ from __future__ import annotations from dataclasses import dataclass, field +import hashlib import re from pathlib import Path from typing import TYPE_CHECKING @@ -20,7 +21,10 @@ from .discovery import ( Component, EMPTY_SHA256, + LinkInput, component_of, + owned_input, + resolve_static_archive, resolve_system_library, resolve_system_library_by_build_id, sha256_file, @@ -34,6 +38,13 @@ FIRST_PARTY = "__first_party__" +# How a component that reached no file got into the build, by kind. Checked-in +# sources were never fetched, and the collector's own vocabulary says so. +_ARRIVED = { + "vendored-source": "checked into this repository", + "vcpkg-port": "installed by vcpkg for this build", +} + # Text-ish members worth reading a REUSE tag out of. _TAGGABLE = { ".py", @@ -92,6 +103,8 @@ class Inventory: external_runtime: dict[str, list[str]] requires_dist: list[str] unattributed: list[str] = field(default_factory=list) + # Link-line inputs nothing explains, as ` (linked by ...): `. + untraced: list[str] = field(default_factory=list) absent_components: dict[str, str] = field(default_factory=dict) _indexed: tuple[dict, dict] | None = field(default=None, repr=False) @@ -125,6 +138,11 @@ class Resolver: def __init__(self, discovery: Discovery, system_resolver=None) -> None: self._system_resolver = system_resolver or resolve_system_library self.system_libraries: dict[str, dict] = {} + # Archives and objects this wheel's members link that no target + # produced, by path, with what explains each -- or `explained: False` + # for the build to refuse. + self.static_inputs: dict[str, dict] = {} + self._input_keys: dict[str, str] = {} self.repo_root = discovery.repo_root self.build_dir = discovery.build_dir self.graph = discovery.graph @@ -155,7 +173,16 @@ def __init__(self, discovery: Discovery, system_resolver=None) -> None: ) def _from_artifact(self, artifact, path: str, detail: str) -> Attribution: + # What a fetched tree, a vcpkg port or vendored material holds is + # credited there; what is left is the build machine's, or the gate's. contributions = self.graph.contributions(artifact, self.ownership) + linked = self.graph.static_inputs(artifact) + for item in sorted(linked, key=lambda item: (item.key, item.named)): + if owned_input(item, self.ownership): + continue + key = self._static_input(item, linked[item]) + if key: + contributions.setdefault(key, set()).add("prebuilt-library") first_party = self.graph.first_party(artifact, self.repo_root, self.ownership) primary = None if not first_party: @@ -175,6 +202,84 @@ def _from_artifact(self, artifact, path: str, detail: str) -> Attribution: primary=primary, ) + def _static_input(self, item: LinkInput, targets: set[str]) -> str | None: + """The component a linked archive or object belongs to, if any.""" + record = self.static_inputs.get(item.key) + if record is None: + record = self.static_inputs[item.key] = self._input_record(item) + record["named_as"] = sorted({*record["named_as"], item.named}) + record["linked_by"] = sorted({*record["linked_by"], *targets}) + return record["component"] + + def _input_record(self, item: LinkInput) -> dict: + """What explains one input the ownership index does not: the repository, + the host, or nothing -- which the build refuses.""" + record: dict = { + "kind": item.kind, + "named_as": [], + "linked_by": [], + "component": None, + "explained": True, + } + + def refused(outcome: str) -> dict: + record["explained"] = False + record["outcome"] = outcome + return record + + if item.path is None or item.kind == "unresolved": + return refused(item.reason) + path = item.path + if item.kind == "archive": + record["members"] = item.members + if item.members == 0: + record["outcome"] = "contributes no code: the archive has no members" + return record + if path.is_relative_to(self.build_dir.resolve()): + return refused( + "inside the build tree, but no target produced it and nothing " + "records which component installed it" + ) + if self.repo_files.digest_of(path): + # REUSE names a holder for every tracked file, and the vendored + # scan registers one that is not this project. + record["outcome"] = "tracked in this repository, held by this project" + return record + + origin = resolve_static_archive(path) + record.update(origin) + if not (origin["package"] or origin["toolkit_root"]): + return refused( + "no package on the build machine owns it and no known toolkit holds it" + if origin["resolved_path"] + else "not found on the build machine" + ) + component = system_component( + path.name, + origin, + route=f"Statically linked from {path}.", + key=self._input_key(path), + ) + self.components[component.key] = component + record["component"] = component.key + record["outcome"] = ( + f"supplied by system package {origin['package']}" + if origin["package"] + else f"part of the toolkit at {origin['toolkit_root']}" + ) + return record + + def _input_key(self, path: Path) -> str: + """`system:`, unless another linked file already holds that name. + + A shared library's key is its SONAME, which no archive or object shares. + """ + key = f"system:{path.name}" + if self._input_keys.get(key, str(path)) != str(path): + key += "@" + hashlib.sha256(str(path).encode()).hexdigest()[:8] + self._input_keys[key] = str(path) + return key + def resolve( self, wheel_path: str, @@ -620,14 +725,8 @@ def build( for key in sorted(roles) if key in resolver.components } - # Say how the component got here. Checked-in sources were never fetched, and - # the collector's own vocabulary for them elsewhere says so. absent = { - key: ( - "checked into this repository" - if resolver.components[key].kind == "vendored-source" - else "fetched for this build" - ) + key: _ARRIVED.get(resolver.components[key].kind, "fetched for this build") + "; contributes to no file in this wheel" for key in sorted(resolver.components) if key not in roles @@ -649,6 +748,11 @@ def build( }, requires_dist=sorted(metadata.get("requires_dist") or []), unattributed=sorted(unattributed), + untraced=sorted( + f"{path} (linked by {', '.join(record['linked_by'])}): {record['outcome']}" + for path, record in resolver.static_inputs.items() + if not record["explained"] + ), absent_components=absent, ) diff --git a/scripts/sbom/sbom/licensing.py b/scripts/sbom/sbom/licensing.py index d697f3bd23..a1dc513f54 100644 --- a/scripts/sbom/sbom/licensing.py +++ b/scripts/sbom/sbom/licensing.py @@ -74,7 +74,9 @@ class LicenseEvidence: """A license text that was read out of the component, not assumed.""" component: str - origin: str # "component-file" | "archive-member" | "readme-section" | "build-host" + # "component-file" | "archive-member" | "readme-section" | "build-host" + # | "vcpkg-port" + origin: str path: str sha256: str size: int @@ -892,6 +894,37 @@ def read_notices(text: str) -> list[str]: return found +_NO_HOLDER = frozenset({"none", "n/a", "unknown"}) + + +def _copyright_fields(lines: list[str]) -> list[list[list[str]]]: + """Each Debian `Copyright:` field's paragraphs, each a list of its lines. + + The field runs on over indented lines and ` .` stands for a blank one, + which separates paragraphs. Debian writes `None` where a package states no + holder, and what follows that is commentary, not a holder. + """ + fields: list[list[list[str]]] = [] + for index, line in enumerate(lines): + if not line.lower().startswith("copyright:"): + continue + raw = [line[len("copyright:") :]] + for following in lines[index + 1 :]: + if not following[:1].isspace() or not following.strip(): + break + raw.append(following) + paragraphs: list[list[str]] = [[]] + for value in (" ".join(item.split()) for item in raw): + if value.lower() in _NO_HOLDER: + break + if value == ".": + paragraphs.append([]) + elif value: + paragraphs[-1].append(value) + fields.append([paragraph for paragraph in paragraphs if paragraph]) + return fields + + def _debian_copyright_fields(lines: list[str]) -> list[str]: """Holders a Debian copyright file states without the word `Copyright`. @@ -900,26 +933,15 @@ def _debian_copyright_fields(lines: list[str]) -> list[str]: in the file names them, so a parser looking for the word finds nobody. """ found: list[str] = [] - for index, line in enumerate(lines): - if not line.startswith("Copyright:"): - continue - # Debian writes the field either way, and the same-line form is the - # commoner by far. `Copyright: Jane Doe` matches no notice pattern -- - # the colon breaks it -- so without this such a package credits nobody. - inline = line[len("Copyright:") :].strip() - values = ([f" {inline}"] if inline else []) + list(lines[index + 1 :]) + for field in _copyright_fields(lines): + values = [value for paragraph in field for value in paragraph] for position, value in enumerate(values): - if not value[:1].isspace() or not value.strip(): - break - holder = " ".join(value.split()).rstrip(".") + holder = value.rstrip(".") # A field whose value opens with a notice proper is read by the scan # below, prose and all; only a field naming a bare holder is read # here, and only for as long as it keeps naming one. if _PLAIN_COPYRIGHT.match(holder) or _PROSE.search(holder): break - # Debian writes `None` where a package states no holder. - if holder.lower() in {"none", "n/a", "unknown"}: - break if position and len(holder) > 60: break if _names_a_holder(f"Copyright {holder}") and holder not in found: @@ -927,6 +949,106 @@ def _debian_copyright_fields(lines: list[str]) -> list[str]: return found +def dep5_copyright(text: str) -> list[str] | None: + """The notices in the `Copyright:` fields of a machine-readable Debian + copyright file; None where the file is not one (no `Format:` field opening + it). + + Only the fields are read: licence paragraphs quote other people's notices. + Stanzas covering only the Debian packaging name its packagers, not the + library's holders. + """ + lines = text.splitlines() + first = next((line for line in lines if line.strip()), "") + if not first.lower().startswith("format:"): + return None + return [ + notice + for stanza in _stanzas(lines) + if not _packaging_only(stanza) + for field in _copyright_fields(stanza) + for paragraph in field + for notice in _notices_in(paragraph) + ] + + +_OPENS_NOTICE = re.compile(r"(?:copyright\b|©|\(c\))", re.IGNORECASE) +_LEADING_YEAR = re.compile(r"\d{4}") +# Markers, years and punctuation only: a notice that names no holder yet. +_NO_HOLDER_YET = re.compile(r"^(?:copyright\b|\(c\)|©|\d{4}|[\s,\-–;/.])*$", re.I) + + +def _notices_in(paragraph: list[str]) -> list[str]: + """A field paragraph's lines, grouped into the notices they state. + + `Copyright`, © or (c) starts a notice, and so does a year -- unless the + notice above names no holder yet, or is a sentence left unfinished -- and + a bare name after a bare name. Any other line continues the notice above. + """ + + def marked(text: str) -> bool: + return bool(_OPENS_NOTICE.match(text) or _LEADING_YEAR.match(text)) + + def bare(text: str) -> bool: + return not ("\n" in text or marked(text) or _PROSE.search(text)) + + def unfinished(current: str) -> bool: + first = current.split("\n", 1)[0] + return ( + not marked(first) + and bool(_PROSE.search(first)) + and not current.endswith(".") + ) + + def opens(line: str, current: str) -> bool: + if _OPENS_NOTICE.match(line): + return True + if _LEADING_YEAR.match(line): + return not (_NO_HOLDER_YET.match(current) or unfinished(current)) + if current.endswith(",") and line[:1].islower(): + return False + return bare(current) and bare(line) + + notices: list[str] = [] + for line in paragraph: + if not notices or opens(line, notices[-1]): + notices.append(line) + else: + notices[-1] += "\n" + line + return notices + + +def _stanzas(lines: list[str]) -> list[list[str]]: + """A Debian control-style file's stanzas, split at blank lines.""" + stanzas: list[list[str]] = [[]] + for line in lines: + if line.strip(): + stanzas[-1].append(line) + elif stanzas[-1]: + stanzas.append([]) + return [stanza for stanza in stanzas if stanza] + + +def _packaging_only(stanza: list[str]) -> bool: + """Whether a stanza's `Files:` patterns all lie under `debian/`. + + Those name whoever packaged the library for Debian, not its holders. A + stanza with no `Files:` field -- the header -- is not one. + """ + patterns: list[str] = [] + for index, line in enumerate(stanza): + if line.lower().startswith("files:"): + patterns += line[len("files:") :].split() + for following in stanza[index + 1 :]: + if not following[:1].isspace(): + break + patterns += following.split() + patterns = [item.removeprefix("./") for item in patterns] + return bool(patterns) and all( + item == "debian" or item.startswith("debian/") for item in patterns + ) + + def _without_dangling(notice: str) -> str: """Drop the connectives a notice ends on because its line broke mid-clause. diff --git a/scripts/sbom/sbom/report.py b/scripts/sbom/sbom/report.py index 76f76702a7..dc7b20979a 100644 --- a/scripts/sbom/sbom/report.py +++ b/scripts/sbom/sbom/report.py @@ -35,6 +35,7 @@ "archive": "fetched archive", "system-library": "build machine", "vendored-source": "vendored source", + "vcpkg-port": "vcpkg port", } diff --git a/tests/python/sbom/synth.py b/tests/python/sbom/synth.py index 5484e66b7a..0802772419 100644 --- a/tests/python/sbom/synth.py +++ b/tests/python/sbom/synth.py @@ -18,6 +18,7 @@ from __future__ import annotations import base64 +import contextlib import csv import gzip import hashlib @@ -593,6 +594,234 @@ def _file_api_reply(root: Path, build: Path, extension: str) -> None: ) +@contextlib.contextmanager +def _edit_json(path: Path): + """Read a reply file, let the caller change it, write it back.""" + payload = json.loads(path.read_text(encoding="utf-8")) + yield payload + path.write_text(json.dumps(payload, indent=1), encoding="utf-8") + + +def _reply(build: Path) -> Path: + return build / ".cmake" / "api" / "v1" / "reply" + + +def _edit_target(build: Path, target: str): + return _edit_json(_reply(build) / f"target-{target}-{CONFIG}.json") + + +def _edit_index(build: Path): + return _edit_json(next(_reply(build).glob("index-*.json"))) + + +def link_line( + build: Path, + target: str, + fragments: tuple[str | tuple[str, str], ...], + *, + dependencies: tuple[str, ...] = (), + directory: str | None = None, + sysroot: Path | None = None, +) -> None: + """Give a fixture target the link line CMake records for a linkable target. + + A fragment is its text, filed under `libraries`, or (text, role). + `directory` is the target's build directory, which is what a relative + fragment is relative to under the Makefile generators. `sysroot` is + CMAKE_SYSROOT, as the reply records it. + """ + with _edit_target(build, target) as payload: + payload["link"] = { + "language": "CXX", + "commandFragments": [ + {"fragment": item, "role": "libraries"} + if isinstance(item, str) + else {"fragment": item[0], "role": item[1]} + for item in fragments + ], + } + if sysroot is not None: + payload["link"]["sysroot"] = {"path": str(sysroot)} + payload["dependencies"] += [{"id": item} for item in dependencies] + if directory is not None: + payload["paths"] = {"build": directory, "source": directory} + + +def set_toolchains( + build: Path, + link_directories: dict[str, list[str] | None], + compilers: dict[str, str] | None = None, +) -> None: + """A toolchains reply: each language's compiler and implicit link directories. + + None for a language's directories is what CMake writes for a language the + top-level project does not enable. + """ + compilers = compilers or {} + toolchains = [] + for language, directories in link_directories.items(): + compiler: dict = { + "implicit": {} if directories is None else {"linkDirectories": directories} + } + if language in compilers: + compiler["path"] = compilers[language] + toolchains.append({"language": language, "compiler": compiler}) + write( + _reply(build) / "toolchains-v1-fixture.json", + json.dumps( + { + "kind": "toolchains", + "version": {"major": 1, "minor": 0}, + "toolchains": toolchains, + } + ), + ) + with _edit_index(build) as index: + index["objects"].append( + { + "kind": "toolchains", + "version": {"major": 1, "minor": 0}, + "jsonFile": "toolchains-v1-fixture.json", + } + ) + + +def add_include(build: Path, target: str, directory: Path) -> None: + """Put one more directory on a fixture target's include path.""" + with _edit_target(build, target) as payload: + payload["compileGroups"][0]["includes"].append({"path": str(directory)}) + + +def set_type(build: Path, target: str, kind: str) -> None: + """Change a fixture target's CMake type.""" + with _edit_target(build, target) as payload: + payload["type"] = kind + + +def set_generator(build: Path, name: str) -> None: + """Record the generator in the file API index, as a real configure does.""" + with _edit_index(build) as index: + index["cmake"] = {"generator": {"name": name, "multiConfig": False}} + + +def vcpkg_port( + installed: Path, + port: str, + version: str, + files: dict[str, bytes], + licence: str, + source: str | tuple[str, ...], + triplet: str = "x64-linux", +) -> None: + """A port as vcpkg installs it: its files, its install list and its SBOM. + + Shaped after what vcpkg 2026-04-08 writes for fmt 11.0.2#1. + """ + base = version.partition("#")[0] + for relative, data in files.items(): + write(installed / triplet / relative, data) + share = installed / triplet / "share" / port + write(share / "copyright", licence) + write( + share / "vcpkg.spdx.json", + json.dumps( + { + "spdxVersion": "SPDX-2.2", + "SPDXID": "SPDXRef-DOCUMENT", + "packages": [ + { + "name": port, + "SPDXID": "SPDXRef-port", + "versionInfo": version, + "downloadLocation": "git+https://github.com/Microsoft/vcpkg@07a73a7565e5de9eb42e90c16c133bdfdfebbcda", + "homepage": f"https://github.com/{port}lib/{port}", + }, + { + "name": f"{port}:{triplet}", + "SPDXID": "SPDXRef-binary", + "downloadLocation": "NONE", + }, + *( + { + "SPDXID": f"SPDXRef-resource-{index}", + "name": item, + "downloadLocation": f"git+https://github.com/{item}@{base}", + } + for index, item in enumerate( + (source,) if isinstance(source, str) else source + ) + ), + ], + } + ), + ) + listed = sorted( + {f"{triplet}/{item}" for item in files} + | { + f"{triplet}/share/{port}/copyright", + f"{triplet}/share/{port}/vcpkg.spdx.json", + } + ) + write( + installed / "vcpkg" / "info" / f"{port}_{base}_{triplet}.list", + "".join(f"{line}\n" for line in [f"{triplet}/", *listed]), + ) + + +def stub_compiler(directory: Path, libraries: Path, sysroot: Path, search: str) -> Path: + """A compiler driver that answers the questions the collector asks one. + + Its linker lists `search` as a SEARCH_DIR under the sysroot, `=` and all. + """ + linker = directory / "ld" + write( + linker, + f'#!/bin/sh\necho \'SEARCH_DIR("={search}"); SEARCH_DIR("/nonexistent")\'\n', + ) + compiler = directory / "cc" + write( + compiler, + "#!/bin/sh\n" + 'case "$1" in\n' + f' -print-search-dirs) echo "libraries: ={libraries}" ;;\n' + f' -print-prog-name=ld) echo "{linker}" ;;\n' + f' -print-sysroot) echo "{sysroot}" ;;\n' + "esac\n", + ) + for path in (linker, compiler): + path.chmod(0o755) + return compiler + + +def elf_object() -> bytes: + """A relocatable ELF, the kind a compiler writes and an archive holds.""" + image = bytearray(elf_shared_object("object.o")) + struct.pack_into(" bytes: + """One `ar` member header, its name written as given.""" + return ( + name.ljust(16) + + b"0".ljust(12) + + b"0".ljust(6) + + b"0".ljust(6) + + b"644".ljust(8) + + str(size).encode().ljust(10) + + b"`\n" + ) + + +def ar_archive(members: dict[str, bytes]) -> bytes: + """A static archive in the GNU `ar` format, holding `members`.""" + image = bytearray(b"!\n") + for name, data in members.items(): + image += ar_header(f"{name}/".encode(), len(data)) + image += data + b"\n" * (len(data) % 2) + return bytes(image) + + # ============================================================================== # Wheel # ============================================================================== diff --git a/tests/python/sbom/test_discovery.py b/tests/python/sbom/test_discovery.py index 9dee044c72..4a68be1c1f 100644 --- a/tests/python/sbom/test_discovery.py +++ b/tests/python/sbom/test_discovery.py @@ -228,6 +228,11 @@ def test_the_codemodel_reader_works_on_a_codemodel_cmake_wrote( for how in graph.contributions(artifact).values(): assert how <= {"compiled-source", "header-include"} + # Relative link-line paths resolve against the directory the link ran in; + # resolved against any other, an archive this build made matches no target. + named = {path for node in graph._nodes.values() for path in node.linked} + assert named & set(graph.artifacts), "this project links static libraries it builds" + def test_a_second_build_tree_is_not_mistaken_for_source(workspace, license_data): """A developer's other build directory holds staged copies of the repo. diff --git a/tests/python/sbom/test_licensing.py b/tests/python/sbom/test_licensing.py index 8bb39308db..a4de013648 100644 --- a/tests/python/sbom/test_licensing.py +++ b/tests/python/sbom/test_licensing.py @@ -431,3 +431,105 @@ def test_the_exception_list_is_required(tmp_path, content): with pytest.raises(licensing.CorpusError, match=r"exceptions\.json"): licensing.load_corpus(tmp_path) + + +def test_a_machine_readable_copyright_field_is_read_as_debian_writes_it(): + """Field names ignore case; `None` (or `n/a`) states no holder, and what + follows it in the field is commentary. A notice wrapped over lines stays + one paragraph, its lines in order.""" + text = ( + "Format: https://www.debian.org/doc/packaging-manuals/copyright-format/1.0/\n\n" + "Files: a\ncopyright: 2001 Lower Case Holder\n .\n 2002 Second Holder\n" + "License: MIT\n\n" + "Files: b\nCopyright: None\n Copied from elsewhere\nLicense: public-domain\n\n" + "Files: c\nCopyright: n/a\nLicense: MIT\n\n" + "Files: d\nCopyright:\n Copyright 1989\n The Regents. All rights reserved.\n" + "License: BSD-3-clause\n" + ) + + assert licensing.dep5_copyright(text) == [ + "2001 Lower Case Holder", + "2002 Second Holder", + "Copyright 1989\nThe Regents. All rights reserved.", + ] + + +def test_the_debian_packaging_stanza_names_no_holder_of_the_library(): + """`Files: debian/*` credits whoever packaged it; `*` and the header count.""" + text = ( + "Format: https://www.debian.org/doc/packaging-manuals/copyright-format/1.0/\n" + "Copyright: 2000 Header Holder\n\n" + "Files: *\nCopyright: 2001 Upstream Holder\nLicense: MIT\n\n" + "Files: debian/*\n debian/patches/fix.patch\n" + "Copyright: 2020 Debian Packager\nLicense: MIT\n\n" + "Files:\n debian/*\nCopyright: 2020 Second Packager\nLicense: MIT\n\n" + "Files: debian ./debian/control\nCopyright: 2020 Third Packager\n" + "License: MIT\n\n" + "Files: debian/rules src/mixed.c\nCopyright: 2021 Mixed Stanza Holder\n" + "License: MIT\n" + ) + + assert licensing.dep5_copyright(text) == [ + "2000 Header Holder", + "2001 Upstream Holder", + "2021 Mixed Stanza Holder", + ] + + +def test_a_field_paragraph_is_split_into_the_notices_it_states(): + """A notice wraps, so a line continues the one above unless it opens a + notice of its own -- and one line per holder is the common layout, which + folding needs to see one at a time.""" + text = ( + "Format: https://www.debian.org/doc/packaging-manuals/copyright-format/1.0/\n\n" + "Files: *\nCopyright: Copyright © 1980, 1982\n" + " The Regents of the University. All rights reserved.\n" + " 2004-2024 Guillem Jover\n 2012 Guillem Jover\n .\n" + " Some code is derived from software contributed to Berkeley by\n" + " Somebody, as part of Summer of Code\n 2005 program.\n" + "License: BSD-3-clause\n\n" + "Files: md5.c\nCopyright:\n Colin Plumb\n Todd C. Miller\nLicense: MIT\n" + ) + + notices = licensing.dep5_copyright(text) + + assert notices == [ + "Copyright © 1980, 1982\nThe Regents of the University. All rights reserved.", + "2004-2024 Guillem Jover", + "2012 Guillem Jover", + "Some code is derived from software contributed to Berkeley by\n" + "Somebody, as part of Summer of Code\n2005 program.", + "Colin Plumb", + "Todd C. Miller", + ] + assert "2012 Guillem Jover" not in licensing.fold_notices(notices) + + +@pytest.mark.parametrize( + ("field", "expected"), + [ + # A year list wrapping onto the holder's line is one notice. + ( + "Copyright: © 1994, 1995,\n 1996 Free Software Foundation, Inc.\n", + ["© 1994, 1995,\n1996 Free Software Foundation, Inc."], + ), + # A holder whose name reads like prose does not swallow the next one. + ( + "Copyright: 1999-2022 Python Software Foundation\n" + " 1992-1993 Jean-loup Gailly\n", + ["1999-2022 Python Software Foundation", "1992-1993 Jean-loup Gailly"], + ), + # A line ending in a comma runs on into a lower-case one. + ( + "Copyright:\n Christian Tismer ,\n contributors\n", + ["Christian Tismer ,\ncontributors"], + ), + ], +) +def test_a_notice_is_not_cut_or_merged_at_a_line_break(field, expected): + text = ( + "Format: https://www.debian.org/doc/packaging-manuals/copyright-format/1.0/\n\n" + f"Files: *\n{field}License: MIT\n" + ) + + assert licensing.dep5_copyright(text) == expected diff --git a/tests/python/sbom/test_static_archives.py b/tests/python/sbom/test_static_archives.py new file mode 100644 index 0000000000..646646295c --- /dev/null +++ b/tests/python/sbom/test_static_archives.py @@ -0,0 +1,982 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""What a link line names is a contribution, wherever it came from.""" + +from __future__ import annotations + +import json + +import pytest +import synth +from sbom import build as build_module +from sbom import discovery +from sbom import document as document_module +from sbom import evidence as evidence_module + +EXTENSION = "_ext.cpython-312-x86_64-linux-gnu.so" +STEM = synth.WHEEL_NAME.removesuffix(".whl") +ONE_OBJECT = synth.ar_archive({"runtime.o": b"\x7fELF object"}) +# An agreement that quotes the notices of what it bundles, as the CUDA EULA does. +EULA = ( + synth.PROPRIETARY_LICENSE + + "\nThird-party notices\n\nCopyright (c) 2011 Upstream Compression Ltd.\n" +) + + +def _collect(workspace) -> dict: + out_dir = workspace.root / "sbom" + build_module.build(workspace.root, workspace.build, workspace.wheel, out_dir) + return { + kind: json.loads((out_dir / f"{STEM}.{kind}.json").read_text()) + for kind in ("spdx", "licenses", "build-evidence") + } + + +def _no_package(monkeypatch) -> None: + """The build machine's package manager owns nothing outside its own trees.""" + monkeypatch.setattr( + discovery, "_identify_host_package", lambda library, origin: origin + ) + + +def _packaged(monkeypatch, package, copyright, **fields) -> None: + """dpkg names `package` as the owner of whatever is asked about. + + `package` and `copyright` may be callables of the library's path. + """ + + def owned(library, origin): + def value(item): + return item(library) if callable(item) else item + + return { + **origin, + **fields, + "package": value(package), + "copyright": str(value(copyright)), + } + + monkeypatch.setattr(discovery, "_identify_host_package", owned) + + +def _row(collected: dict, key: str) -> dict: + """The licence report's row for one component.""" + return next( + item for item in collected["licenses"]["components"] if item["component"] == key + ) + + +def _library_dir(root, *names) -> None: + """A directory holding each `lib` as both a shared library and an archive.""" + for name in names: + synth.write(root / f"lib{name}.so", synth.elf_shared_object(f"lib{name}.so")) + synth.write(root / f"lib{name}.a", ONE_OBJECT) + + +def _inputs(workspace, *fragments, **link) -> dict[str, discovery.LinkInput]: + synth.link_line(workspace.build, "ext", fragments, **link) + graph = discovery.BuildGraph(workspace.build, synth.CONFIG) + linked = graph.static_inputs(graph.by_name(EXTENSION)[0]) + return {item.named: item for item in linked} + + +def _package(row: dict, spdx: dict) -> dict: + package_id = document_module.component_package_id(row["component"]) + return next(item for item in spdx["packages"] if item["SPDXID"] == package_id) + + +@pytest.mark.parametrize( + ("data", "expected"), + [ + (synth.ar_archive({}), 0), + (ONE_OBJECT, 1), + (synth.ar_archive({"a.o": b"a", "b.o": b"bb"}), 2), + # A GNU symbol table is an index, not a member. + (b"!\n" + synth.ar_header(b"/", 4) + b"\0\0\0\0" + ONE_OBJECT[8:], 1), + # BSD names a long member, the symbol table among them, inside its data. + ( + b"!\n" + + synth.ar_header(b"#1/20", 24) + + b"__.SYMDEF SORTED\0\0\0\0" + + b"\0\0\0\0" + + ONE_OBJECT[8:], + 1, + ), + # A thin archive stores no member data, only headers. + ( + b"!\n" + + synth.ar_header(b"a.o/", 100) + + synth.ar_header(b"b.o/", 200), + 2, + ), + (b"/* GNU ld script */\nGROUP ( libm.so.6 )\n", None), + ], +) +def test_archive_members_are_counted_from_the_ar_headers(tmp_path, data, expected): + path = tmp_path / "lib.a" + path.write_bytes(data) + + assert discovery.archive_members(path) == expected + + +@pytest.mark.parametrize( + ("generator", "directory", "fragment"), + [ + (None, None, "lib/libdelta.a"), + ("Ninja", "src/ext", "lib/libdelta.a"), + ("Unix Makefiles", "src/ext", "../../lib/libdelta.a"), + ], +) +def test_an_archive_a_target_here_built_maps_to_that_target( + workspace, generator, directory, fragment +): + """Not a declared dependency of ext; its link line is the only evidence.""" + if generator: + synth.set_generator(workspace.build, generator) + synth.link_line(workspace.build, "ext", (fragment,), directory=directory) + + graph = discovery.BuildGraph(workspace.build, synth.CONFIG) + extension = graph.by_name(EXTENSION)[0] + + assert "compiled-source" in graph.contributions(extension)["delta"] + assert graph.static_inputs(extension) == {} + + +def test_an_archive_through_a_static_library_is_on_the_consumers_link_line( + workspace, tmp_path +): + """CMake gives a static library no link line: what it requires is flattened + onto each consumer's, beside the static library's own archive.""" + runtime = tmp_path / "host" / "libruntime.a" + synth.write(runtime, ONE_OBJECT) + synth.link_line( + workspace.build, + "ext", + ("lib/libdelta.a", str(runtime)), + dependencies=("delta",), + ) + + graph = discovery.BuildGraph(workspace.build, synth.CONFIG) + extension = graph.by_name(EXTENSION)[0] + + assert "delta" in graph.contributions(extension) + assert graph.static_inputs(extension) == { + discovery.LinkInput(str(runtime), runtime, "archive", members=1): {"ext"} + } + + +def test_an_archive_a_dependency_links_is_followed_like_its_sources( + workspace, tmp_path +): + """ext -> delta (static) -> gamma (shared), whose link line names it.""" + runtime = tmp_path / "host" / "libruntime.a" + synth.write(runtime, ONE_OBJECT) + synth.link_line(workspace.build, "ext", (), dependencies=("delta",)) + synth.link_line(workspace.build, "delta", (), dependencies=("gamma",)) + synth.link_line(workspace.build, "gamma", (str(runtime),)) + + graph = discovery.BuildGraph(workspace.build, synth.CONFIG) + extension = graph.by_name(EXTENSION)[0] + + assert list(graph.static_inputs(extension).values()) == [{"gamma"}] + + +def test_static_code_is_found_in_every_form_a_link_line_takes(workspace, tmp_path): + host = tmp_path / "host dir" + synth.write(host / "libspaced.a", ONE_OBJECT) + synth.write(host / "libwhole.a", ONE_OBJECT) + synth.write(host / "libflagged.a", ONE_OBJECT) + synth.write(host / "libversioned.a.1", ONE_OBJECT) + synth.write(host / "prebuilt.o", synth.elf_object()) + synth.write(host / "libshared.so", synth.elf_shared_object("libshared.so")) + synth.write(host / "exports.map", "{ global: *; };\n") + + inputs = _inputs( + workspace, + f'"{host}/libspaced.a"', + f'"-Wl,--whole-archive,{host}/libwhole.a,--no-whole-archive"', + (f'"{host}/libflagged.a"', "flags"), + f'"{host}/libversioned.a.1"', + f'"{host}/prebuilt.o"', + f'"{host}/libshared.so"', + (f'"-Wl,--version-script,{host}/exports.map"', "flags"), + ("-fuse-ld=gold", "flags"), + ) + + assert {name: item.kind for name, item in inputs.items()} == { + f"{host}/libspaced.a": "archive", + f"{host}/libwhole.a": "archive", + f"{host}/libflagged.a": "archive", + f"{host}/libversioned.a.1": "archive", + f"{host}/prebuilt.o": "object", + } + + +def test_a_library_name_resolves_the_way_ld_resolves_it(workspace, tmp_path): + searched = tmp_path / "searched" + implicit = tmp_path / "implicit" + _library_dir(searched, "both", "forced") + synth.write(searched / "libstaticonly.a", ONE_OBJECT) + synth.write(implicit / "libcompiler.a", ONE_OBJECT) + synth.set_toolchains(workspace.build, {"CXX": [str(implicit)]}) + + inputs = _inputs( + workspace, + (f"-L{searched}", "libraryPath"), + "-lboth", + "-lstaticonly", + "-lcompiler", + "-l:libboth.a", + "-lmissing", + "-l:libmissing.a", + "-Wl,-Bstatic", + "-lforced", + ) + + # -lboth takes the shared library, which DT_NEEDED records; after -Bstatic, + # -lforced takes the archive. + assert {name: (item.kind, item.path) for name, item in inputs.items()} == { + "-lforced": ("archive", searched / "libforced.a"), + "-lstaticonly": ("archive", searched / "libstaticonly.a"), + "-lcompiler": ("archive", implicit / "libcompiler.a"), + "-l:libboth.a": ("archive", searched / "libboth.a"), + "-lmissing": ("unresolved", None), + "-l:libmissing.a": ("unresolved", None), + } + + +def test_every_spelling_of_a_search_resolves_alike(workspace, tmp_path): + joined, separated = tmp_path / "joined", tmp_path / "separated" + _library_dir(joined, "eqsign", "spaced", "linkerspaced") + _library_dir(separated, "fromsep") + synth.write(joined / "libarchive.a.1", ONE_OBJECT) + synth.write(joined / "libshared.so.1", synth.elf_shared_object("libshared.so.1")) + synth.write(tmp_path / "xl" / "libxl.a", ONE_OBJECT) + + inputs = _inputs( + workspace, + f"-Wl,--library-path={joined}", + f"-Wl,-L,{separated}", + "-Wl,-Bstatic", + "-Wl,--library=eqsign", + "-l spaced", + "-Wl,-l,linkerspaced", + "-lfromsep", + "-Wl,-Bdynamic", + "-l:libarchive.a.1", + "-l:libshared.so.1", + f"-Xlinker {tmp_path}/xl/libxl.a", + ) + + assert {name: item.path for name, item in inputs.items()} == { + "-leqsign": joined / "libeqsign.a", + "-lspaced": joined / "libspaced.a", + "-llinkerspaced": joined / "liblinkerspaced.a", + "-lfromsep": separated / "libfromsep.a", + # Named outright, so judged by content: an archive without the suffix + # is still one, and a shared library is left to DT_NEEDED. + "-l:libarchive.a.1": joined / "libarchive.a.1", + f"{tmp_path}/xl/libxl.a": tmp_path / "xl" / "libxl.a", + } + + +@pytest.mark.parametrize( + ("fragments", "expected"), + [ + # -Bdynamic resets; push-state saves and pop-state restores. + ( + ( + "-Wl,-Bstatic", + "-Wl,-Bdynamic", + "-lreset", + "-Wl,--push-state,-Bstatic,-lpushed,--pop-state", + "-lafter", + ), + {"-lpushed": "libpushed.a"}, + ), + # Pushed while static: the pop restores static mode, not the default. + ( + ( + "-Wl,-Bstatic", + "-Wl,--push-state,-Bdynamic,-lpushdyn,--pop-state", + "-lpopped", + ), + {"-lpopped": "libpopped.a"}, + ), + # The driver's -static covers the whole line, wherever it is written. + (("-lafter", ("-static", "flags")), {"-lafter": "libafter.a"}), + ], +) +def test_static_mode_follows_ld_and_the_driver( + workspace, tmp_path, fragments, expected +): + directory = tmp_path / "libs" + _library_dir(directory, "reset", "pushed", "after", "pushdyn", "popped") + + inputs = _inputs(workspace, f"-L{directory}", *fragments) + + assert {name: item.path for name, item in inputs.items()} == { + name: directory / file for name, file in expected.items() + } + + +def test_an_option_value_is_not_an_input(workspace, tmp_path): + synth.write(tmp_path / "libvalue.a", ONE_OBJECT) + + inputs = _inputs( + workspace, + (f"-o {tmp_path}/libvalue.a", "flags"), + f"-Wl,--dependency-file,{tmp_path}/deps.d", + f"-Wl,--require-defined,{tmp_path}/libvalue.a", + (f"-Wl,-rpath,{tmp_path}", "flags"), + ) + + assert inputs == {} + + +def test_an_unknown_path_names_itself_and_the_option_it_may_belong_to(workspace): + (item,) = _inputs(workspace, "-Wl,--some-new-option,mystery-value").values() + + assert item.kind == "unresolved" + assert item.named == "mystery-value" + assert "linker option" in item.reason + + +def test_the_compiler_and_linker_are_asked_where_the_reply_is_silent( + workspace, tmp_path +): + """C enabled only below the top level gets no implicit directories from + CMake, so the compiler is asked, and then its linker's SEARCH_DIRs.""" + from_compiler = tmp_path / "compiler" + from_linker = tmp_path / "sysroot" / "stub-only" / "lib" + synth.write(from_compiler / "libfromcompiler.a", ONE_OBJECT) + synth.write(from_linker / "libfromlinker.a", ONE_OBJECT) + compiler = synth.stub_compiler( + tmp_path / "bin", from_compiler, tmp_path / "sysroot", "/stub-only/lib" + ) + synth.set_toolchains(workspace.build, {"CXX": None}, {"CXX": str(compiler)}) + + inputs = _inputs(workspace, "-lfromcompiler", "-lfromlinker") + + assert {name: item.path for name, item in inputs.items()} == { + "-lfromcompiler": from_compiler / "libfromcompiler.a", + "-lfromlinker": from_linker / "libfromlinker.a", + } + + +def test_a_search_directory_under_the_sysroot_is_read_there(workspace, tmp_path): + """`-L=/dir` names `/dir`, not a directory called `=`.""" + sysroot = tmp_path / "sysroot" + synth.write(sysroot / "sub" / "libinroot.a", ONE_OBJECT) + synth.write(tmp_path / "compiler" / "libinroot.a", ONE_OBJECT) + compiler = synth.stub_compiler( + tmp_path / "bin", tmp_path / "compiler", sysroot, "/stub-only/lib" + ) + synth.set_toolchains(workspace.build, {"CXX": None}, {"CXX": str(compiler)}) + + (item,) = _inputs(workspace, "-L=/sub", "-linroot").values() + + assert item.path == sysroot / "sub" / "libinroot.a" + + +def test_a_script_in_the_sysroot_names_files_in_the_sysroot(workspace, tmp_path): + """libc.so in a sysroot names `/usr/lib/...`, which ld reads under it.""" + sysroot = tmp_path / "sysroot" + synth.write(sysroot / "lib" / "libroot_nonshared.a", ONE_OBJECT) + synth.write(sysroot / "lib" / "libprefixed.a", ONE_OBJECT) + synth.write( + sysroot / "lib" / "libroot.so", + "/* GNU ld script */\nGROUP ( /lib/libroot_nonshared.a =/lib/libprefixed.a )\n", + ) + compiler = synth.stub_compiler( + tmp_path / "bin", tmp_path / "compiler", sysroot, "/stub-only/lib" + ) + synth.set_toolchains(workspace.build, {"CXX": None}, {"CXX": str(compiler)}) + + synth.link_line(workspace.build, "ext", (f"-L{sysroot}/lib", "-lroot")) + graph = discovery.BuildGraph(workspace.build, synth.CONFIG) + inputs = graph.static_inputs(graph.by_name(EXTENSION)[0]) + + assert sorted(item.path for item in inputs) == [ + sysroot / "lib" / "libprefixed.a", + sysroot / "lib" / "libroot_nonshared.a", + ] + + +def test_cmake_sysroot_overrides_the_compilers_own(workspace, tmp_path): + """CMAKE_SYSROOT reaches the linker as --sysroot, for `-L=` and for ld's + `=` search directories alike.""" + compiler_root, cmake_root = tmp_path / "compiler-root", tmp_path / "cmake-root" + for root in (compiler_root, cmake_root): + synth.write(root / "sub" / "libfoo.a", ONE_OBJECT) + synth.write(root / "stub-only" / "lib" / "libfromld.a", ONE_OBJECT) + compiler = synth.stub_compiler( + tmp_path / "bin", tmp_path / "compiler", compiler_root, "/stub-only/lib" + ) + synth.set_toolchains(workspace.build, {"CXX": None}, {"CXX": str(compiler)}) + + inputs = _inputs(workspace, "-L=/sub", "-lfoo", "-lfromld", sysroot=cmake_root) + + assert {name: item.path for name, item in inputs.items()} == { + "-lfoo": cmake_root / "sub" / "libfoo.a", + "-lfromld": cmake_root / "stub-only" / "lib" / "libfromld.a", + } + + +@pytest.mark.parametrize( + ("fragments", "expected"), + [ + (("-Wl,-Bstatic", "-l:libgrp.so"), ["libz.a"]), + (("-Wl,-Bstatic", "{d}/libgrp.so"), ["libz.a"]), + (("-Wl,-Bstatic", "-l:libouter.so"), ["libz.a"]), + (("-lgrp",), []), + ], +) +def test_static_mode_applies_inside_a_linker_script( + workspace, tmp_path, fragments, expected +): + _library_dir(tmp_path, "z") + synth.write(tmp_path / "libgrp.so", "/* GNU ld script */\nGROUP ( -lz )\n") + synth.write(tmp_path / "libouter.so", "/* GNU ld script */\nINPUT ( libgrp.so )\n") + + inputs = _inputs( + workspace, f"-L{tmp_path}", *(item.format(d=tmp_path) for item in fragments) + ) + + assert [item.path.name for item in inputs.values()] == expected + + +def test_no_compiler_on_record_says_to_reconfigure(workspace): + cache = workspace.build / "CMakeCache.txt" + cache.write_text( + "".join( + line + for line in cache.read_text().splitlines(keepends=True) + if "_COMPILER:" not in line + ) + ) + + (item,) = _inputs(workspace, "-lanything").values() + + assert "reconfigure" in item.reason + + +def test_a_linker_script_is_followed_to_what_it_names(workspace, tmp_path): + """libc.so names libc_nonshared.a beside libc.so.6; glibc's libm.a is a + script naming libm-.a.""" + directory = tmp_path / "libs" + synth.write(directory / "libscripted.so.6", synth.elf_shared_object("x.so.6")) + synth.write(directory / "libscripted_nonshared.a", ONE_OBJECT) + synth.write(directory / "libinner.a", ONE_OBJECT) + synth.write( + directory / "libscripted.so", + "/* GNU ld script */\nOUTPUT_FORMAT(elf64-x86-64)\n" + f"GROUP ( {directory}/libscripted.so.6 {directory}/libscripted_nonshared.a" + f" AS_NEEDED ( {directory}/libscripted.so.6 ) )\nINPUT(-linner)\n", + ) + synth.write(directory / "libm-9.99.a", ONE_OBJECT) + synth.write(directory / "libmath.a", "/* GNU ld script */\nGROUP ( libm-9.99.a )\n") + # ld looks beside the script first, then where the link runs, then -L. + elsewhere = tmp_path / "elsewhere" + synth.write(directory / "libbeside.a", ONE_OBJECT) + synth.write(elsewhere / "libbeside.a", ONE_OBJECT) + synth.write(workspace.build / "libbeside.a", ONE_OBJECT) + synth.write(workspace.build / "libcwd.a", ONE_OBJECT) + synth.write( + elsewhere / "libnear.so", + "/* GNU ld script */\nGROUP ( libbeside.a libcwd.a )\n", + ) + + synth.link_line( + workspace.build, + "ext", + (f"-L{directory}", "-lscripted", "-l:libmath.a", f"{elsewhere}/libnear.so"), + ) + graph = discovery.BuildGraph(workspace.build, synth.CONFIG) + inputs = graph.static_inputs(graph.by_name(EXTENSION)[0]) + + assert sorted((item.named, item.path) for item in inputs) == sorted( + [ + (f"{elsewhere}/libnear.so", elsewhere / "libbeside.a"), + (f"{elsewhere}/libnear.so", (workspace.build / "libcwd.a").resolve()), + ("-l:libmath.a", directory / "libm-9.99.a"), + ("-lscripted", directory / "libinner.a"), + ("-lscripted", directory / "libscripted_nonshared.a"), + ] + ) + + +def test_a_script_naming_itself_says_so(workspace, tmp_path): + synth.write(tmp_path / "libself.so", "/* GNU ld script */\nGROUP ( libself.so )\n") + + inputs = _inputs(workspace, f"-L{tmp_path}", "-lself") + + assert [item.reason for item in inputs.values()] == [ + "linker scripts nest too deeply here" + ] + + +@pytest.mark.parametrize("kind", ["EXECUTABLE", "MODULE_LIBRARY"]) +def test_a_build_order_dependency_links_nothing_into_the_member(workspace, kind): + """A code generator ext waits for is not linked into it, so neither its line + nor a shared library reached only through it is.""" + synth.set_type(workspace.build, "unit_tests", kind) + synth.link_line(workspace.build, "gamma", ("-l:libviagamma.a",)) + synth.link_line( + workspace.build, "unit_tests", ("-l:libnothing.a",), dependencies=("gamma",) + ) + synth.link_line(workspace.build, "ext", (), dependencies=("unit_tests",)) + + graph = discovery.BuildGraph(workspace.build, synth.CONFIG) + + assert graph.static_inputs(graph.by_name(EXTENSION)[0]) == {} + assert { + item.named for item in graph.static_inputs(graph.by_name("unit_tests")[0]) + } == {"-l:libnothing.a", "-l:libviagamma.a"} + + +def test_a_fragment_that_does_not_parse_is_unresolved_not_dropped(workspace): + inputs = _inputs(workspace, '"/opt/lib/libunbalanced.a') + + (item,) = inputs.values() + assert item.kind == "unresolved" + assert "does not parse" in item.reason + + +def _link(workspace, *fragments) -> None: + synth.link_line(workspace.build, "ext", fragments) + + +def test_a_packaged_archive_is_inventoried_through_its_package( + workspace, tmp_path, monkeypatch +): + archive = tmp_path / "host" / "usr" / "lib" / "libsquash.a" + synth.write(archive, ONE_OBJECT) + copyright_file = tmp_path / "host" / "copyright" + synth.write(copyright_file, synth.spdx_text("Zlib")) + _packaged( + monkeypatch, + "libsquash-dev", + copyright_file, + supplier="Organization: Example Distro (devel@example.com)", + version="1.0-1", + distro="ubuntu", + ) + _link(workspace, str(archive)) + + collected = _collect(workspace) + + row = _row(collected, "system:libsquash.a") + assert row["license_concluded"] == "Zlib" + assert row["how_it_reaches_the_wheel"] == ["prebuilt-library"] + assert row["purl"] == "pkg:deb/ubuntu/libsquash-dev@1.0-1" + assert row["source_info"].startswith(f"Statically linked from {archive}.") + evidence = collected["build-evidence"] + record = evidence["static_inputs"][str(archive)] + assert record["package"] == "libsquash-dev" + assert record["linked_by"] == ["ext"] + linked = next( + item for item in evidence["link_artifacts"] if item["target"] == "ext" + ) + assert linked["static_inputs"] == [str(archive)] + + +# A machine-readable copyright file whose licence paragraph quotes a notice. +DEP5 = ( + "Format: https://www.debian.org/doc/packaging-manuals/copyright-format/1.0/\n" + "Upstream-Name: squash\n\n" + "Files: *\nCopyright: 2001-2020 Squash Authors\n" + " 2019 Example Corp\nLicense: Squash\n\n" + "License: Squash\n Permission is granted to use this software freely.\n" + " Copyright (c) 1995 Someone Quoted In The Text\n" +) + + +@pytest.mark.parametrize( + ("text", "expected"), + [ + # Not machine-readable: the notices it quotes belong to what it bundles. + (EULA, "NOASSERTION"), + (DEP5, "2001-2020 Squash Authors\n2019 Example Corp"), + # Free-form, with a `Copyright:` heading at the start of a line. + ( + "This package was debianized by someone.\n\n" + "Copyright: 2001 Quoted Upstream\n", + "NOASSERTION", + ), + ], +) +def test_a_system_component_names_only_the_holders_its_package_declares( + workspace, tmp_path, monkeypatch, text, expected +): + archive = tmp_path / "host" / "libsquash.a" + synth.write(archive, ONE_OBJECT) + copyright_file = tmp_path / "host" / "copyright" + synth.write(copyright_file, text) + _packaged(monkeypatch, "libsquash-dev", copyright_file) + _link(workspace, str(archive)) + + collected = _collect(workspace) + + row = _row(collected, "system:libsquash.a") + assert _package(row, collected["spdx"])["copyrightText"] == expected + + +def test_a_cuda_archive_is_a_component_under_the_toolkits_terms( + workspace, tmp_path, monkeypatch +): + toolkit = tmp_path / "cuda-99.9" + archive = toolkit / "targets" / "x86_64-linux" / "lib" / "libcudart_static.a" + synth.write(archive, ONE_OBJECT) + synth.write(toolkit / "EULA.txt", EULA) + _no_package(monkeypatch) + monkeypatch.setattr(discovery, "_toolkit_roots", lambda: [toolkit.resolve()]) + _link(workspace, str(archive)) + + collected = _collect(workspace) + + key = "system:libcudart_static.a" + row = _row(collected, key) + assert row["how_it_reaches_the_wheel"] == ["prebuilt-library"] + assert [item["path"] for item in row["evidence"]] == [ + str((toolkit / "EULA.txt").resolve()) + ] + assert row["license_concluded"].startswith("LicenseRef-") + assert _package(row, collected["spdx"])["copyrightText"] == "NOASSERTION" + + member = next( + item + for item in collected["build-evidence"]["attributions"] + if item["path"].endswith(EXTENSION) + ) + assert member["components"][key] == ["prebuilt-library"] + assert { + "spdxElementId": document_module.WHEEL_PACKAGE_ID, + "relatedSpdxElement": document_module.component_package_id(key), + "relationshipType": "STATIC_LINK", + } in collected["spdx"]["relationships"] + + +def test_two_archives_sharing_a_name_are_two_components( + workspace, tmp_path, monkeypatch +): + first = tmp_path / "one" / "libsame.a" + second = tmp_path / "two" / "libsame.a" + for archive in (first, second): + synth.write(archive, ONE_OBJECT) + synth.write(archive.parent / "copyright", synth.spdx_text("Zlib")) + _packaged( + monkeypatch, + lambda library: f"libsame-{library.parent.name}", + lambda library: library.parent / "copyright", + ) + _link(workspace, str(first), str(second)) + + collected = _collect(workspace) + + records = collected["build-evidence"]["static_inputs"] + keys = {records[str(first)]["component"], records[str(second)]["component"]} + assert len(keys) == 2 + assert keys <= {item["component"] for item in collected["licenses"]["components"]} + + +def test_an_archive_with_no_members_is_recorded_not_inventoried( + workspace, tmp_path, monkeypatch +): + archive = tmp_path / "host" / "librt.a" + synth.write(archive, synth.ar_archive({})) + _no_package(monkeypatch) + _link(workspace, str(archive)) + + collected = _collect(workspace) + + record = collected["build-evidence"]["static_inputs"][str(archive)] + assert record["members"] == 0 + assert record["component"] is None + assert "no members" in record["outcome"] + assert not any( + item["component"].endswith("librt.a") + for item in collected["licenses"]["components"] + ) + + +def _tracked_archive(workspace, monkeypatch, holder=None): + """A tracked archive linked into ext, held by `holder` through REUSE.toml.""" + relative = "prebuilt/libtracked.a" + synth.write(workspace.root / relative, ONE_OBJECT) + if holder: + synth.write( + workspace.root / "REUSE.toml", + synth.reuse_toml(synth.annotation(relative, holder)), + ) + synth.track(workspace.root) + _no_package(monkeypatch) + _link(workspace, str(workspace.root / relative)) + return (workspace.root / relative).resolve() + + +def test_a_tracked_archive_is_credited_to_the_holder_reuse_names( + workspace, monkeypatch +): + _tracked_archive(workspace, monkeypatch, "Upstream Archive Co") + + collected = _collect(workspace) + + row = _row(collected, "vendored:upstream-archive-co") + assert "prebuilt-library" in row["how_it_reaches_the_wheel"] + + +def test_a_tracked_archive_naming_no_other_holder_is_the_projects( + workspace, monkeypatch +): + """Every tracked file has REUSE info; one naming nobody else is the project's.""" + archive = _tracked_archive(workspace, monkeypatch) + + collected = _collect(workspace) + + record = collected["build-evidence"]["static_inputs"][str(archive)] + assert record["component"] is None + assert "held by this project" in record["outcome"] + + +@pytest.mark.parametrize("where", ["host", "build", "unresolved"]) +def test_an_input_nothing_explains_stops_the_build( + workspace, tmp_path, monkeypatch, where +): + _no_package(monkeypatch) + monkeypatch.setattr(discovery, "_toolkit_roots", list) + if where == "unresolved": + fragment = "-l:libmystery.a" + else: + root = workspace.build / "prebuilt" if where == "build" else tmp_path / "opt" + synth.write(root / "libmystery.a", ONE_OBJECT) + fragment = str((root / "libmystery.a").resolve()) + _link(workspace, fragment) + + with pytest.raises( + build_module.BuildError, match=r"libmystery\.a \(linked by ext\)" + ): + _collect(workspace) + + +FMT_LICENCE_HEAD = ( + "Copyright (c) 2012 - present, Victor Zverovich and {fmt} contributors\n\n" +) + + +def _fmt_port(installed) -> None: + synth.vcpkg_port( + installed, + "fmt", + "11.0.2#1", + { + "lib/libfmt.a": ONE_OBJECT, + "debug/lib/libfmtd.a": ONE_OBJECT, + "include/fmt/core.h": b"#pragma once\n", + }, + FMT_LICENCE_HEAD + synth.spdx_text("MIT"), + "fmtlib/fmt", + ) + + +@pytest.mark.parametrize("outside_build", [False, True]) +def test_a_vcpkg_archive_is_credited_to_the_port_that_installed_it( + workspace, tmp_path, monkeypatch, outside_build +): + """spdlog built against vcpkg's fmt puts libfmt.a on every consumer's line.""" + installed = ( + tmp_path / "elsewhere" / "vcpkg_installed" + if outside_build + else workspace.build / "vcpkg_installed" + ) + if outside_build: + cache = workspace.build / "CMakeCache.txt" + cache.write_text(cache.read_text() + f"VCPKG_INSTALLED_DIR:PATH={installed}\n") + _fmt_port(installed) + _no_package(monkeypatch) + _link(workspace, str(installed / "x64-linux" / "lib" / "libfmt.a")) + + collected = _collect(workspace) + + row = _row(collected, "vcpkg:fmt") + assert row["kind"] == "vcpkg-port" + assert row["version"] == "11.0.2" + assert row["license_concluded"] == "MIT" + assert row["purl"] == "pkg:github/fmtlib/fmt@11.0.2" + assert row["supplier"] == "Organization: fmtlib" + assert "prebuilt-library" in row["how_it_reaches_the_wheel"] + assert "vcpkg port fmt 11.0.2#1 for x64-linux" in row["source_info"] + package = _package(row, collected["spdx"]) + assert "Victor Zverovich" in package["copyrightText"] + assert { + "spdxElementId": document_module.WHEEL_PACKAGE_ID, + "relatedSpdxElement": package["SPDXID"], + "relationshipType": "STATIC_LINK", + } in collected["spdx"]["relationships"] + + +@pytest.mark.parametrize("target", ["x64-linux", "arm64-linux"]) +def test_a_port_installed_for_two_triplets_owns_both_installs(workspace, target): + """A port the build needs as a host tool and as a library has a list per + triplet. Both installs are its files; the component describes the target's.""" + installed = workspace.build / "vcpkg_installed" + triplets = ("arm64-linux", "x64-linux") + for triplet in triplets: + synth.vcpkg_port( + installed, + "fmt", + "11.0.2#1", + {"lib/libfmt.a": ONE_OBJECT}, + synth.spdx_text("MIT"), + "fmtlib/fmt", + triplet=triplet, + ) + + components, owned = discovery.discover_vcpkg(installed, target) + + assert {installed / triplet / "lib" / "libfmt.a" for triplet in triplets} <= set( + owned["vcpkg:fmt"] + ) + assert components["vcpkg:fmt"].source_info.startswith( + f"Built and installed by vcpkg port fmt 11.0.2#1 for {target}." + ) + + +def test_the_cache_names_the_triplet_a_port_is_described_from(workspace): + installed = workspace.build / "vcpkg_installed" + for triplet in ("arm64-linux", "x64-linux"): + synth.vcpkg_port( + installed, + "fmt", + "11.0.2#1", + {"lib/libfmt.a": ONE_OBJECT}, + synth.spdx_text("MIT"), + "fmtlib/fmt", + triplet=triplet, + ) + cache = workspace.build / "CMakeCache.txt" + cache.write_text(cache.read_text() + "VCPKG_TARGET_TRIPLET:STRING=x64-linux\n") + + space = evidence_module.discover(workspace.root, workspace.build) + + assert "for x64-linux." in space.components["vcpkg:fmt"].source_info + + +def test_a_file_no_port_lists_is_still_unexplained(workspace, monkeypatch): + installed = workspace.build / "vcpkg_installed" + _fmt_port(installed) + stray = installed / "x64-linux" / "lib" / "libstray.a" + synth.write(stray, ONE_OBJECT) + _no_package(monkeypatch) + _link(workspace, str(stray)) + + with pytest.raises(build_module.BuildError, match=r"libstray\.a \(linked by ext\)"): + _collect(workspace) + + +def test_an_unlinked_port_is_excluded_as_installed_by_vcpkg(workspace): + synth.vcpkg_port( + workspace.build / "vcpkg_installed", + "zlib", + "1.3.1", + {"lib/libz.a": ONE_OBJECT}, + synth.spdx_text("Zlib"), + "madler/zlib", + ) + + collected = _collect(workspace) + + exclusion = next( + item + for item in collected["licenses"]["exclusions"] + if item["component"] == "vcpkg:zlib" + ) + assert exclusion["reason"].startswith("installed by vcpkg for this build") + + +def _eigen_port(installed) -> None: + synth.vcpkg_port( + installed, + "eigen3", + "3.4.0#5", + {"include/eigen3/Eigen/Core": b"#pragma once\n"}, + synth.spdx_text("MIT"), + ("libeigen/eigen", "libeigen/eigen-patches"), + ) + + +def test_an_include_directory_one_port_owns_credits_that_port(workspace): + """The fixture's extension also searches the shared include root.""" + installed = workspace.build / "vcpkg_installed" + _fmt_port(installed) + _eigen_port(installed) + synth.add_include( + workspace.build, "ext", installed / "x64-linux" / "include" / "eigen3" + ) + + space = evidence_module.discover(workspace.root, workspace.build) + graph = space.graph + contributions = graph.contributions(graph.by_name(EXTENSION)[0], space.ownership) + + assert contributions["vcpkg:eigen3"] == {"header-include"} + assert "vcpkg:fmt" not in contributions + + +def test_a_port_with_several_sources_names_no_upstream(workspace): + _eigen_port(workspace.build / "vcpkg_installed") + + collected = _collect(workspace) + + row = _row(collected, "vcpkg:eigen3") + assert row["version"] == "3.4.0" + assert row["purl"] == "pkg:generic/eigen3@3.4.0" + assert row["supplier"] == "NOASSERTION" + assert "vcpkg port eigen3 3.4.0#5" in row["source_info"] + + +def test_a_symlinked_script_is_read_beside_the_link(workspace, tmp_path): + """ld resolves a script's relative names beside the name it was given.""" + target, links = tmp_path / "target", tmp_path / "links" + synth.write(target / "libreal.so", "/* GNU ld script */\nGROUP ( libmine.a )\n") + synth.write(target / "libmine.a", ONE_OBJECT) + synth.write(links / "libmine.a", ONE_OBJECT) + (links / "libsym.so").symlink_to(target / "libreal.so") + + (item,) = _inputs(workspace, f"-L{links}", "-lsym").values() + + assert item.path == links / "libmine.a" + + +def test_ownership_sees_a_file_registered_after_a_lookup(tmp_path): + ownership = discovery.Ownership() + include = tmp_path / "include" + ownership.register_file(include / "a" / "a.h", "port-a", by_directory=False) + assert ownership.owners_under(include) == {"port-a"} + + ownership.register_file(include / "b" / "b.h", "port-b", by_directory=False) + ownership.register_file(include / "c.h", "fetched") + + # Two listed owners under one directory credit neither; each alone does. + assert ownership.owners_under(include) == {"fetched"} + assert ownership.owners_under(include / "b") == {"port-b"} + + +def test_an_empty_archive_credits_no_one_however_it_is_held(workspace): + installed = workspace.build / "vcpkg_installed" + synth.vcpkg_port( + installed, + "stubs", + "1.0", + {"lib/libstub.a": synth.ar_archive({})}, + synth.spdx_text("MIT"), + "example/stubs", + ) + _link(workspace, str(installed / "x64-linux" / "lib" / "libstub.a")) + + space = evidence_module.discover(workspace.root, workspace.build) + contributions = space.graph.contributions( + space.graph.by_name(EXTENSION)[0], space.ownership + ) + + assert "vcpkg:stubs" not in contributions From cfcf5cee589dca305c1ac91441ad11dc896af5f6 Mon Sep 17 00:00:00 2001 From: Andrei Aristarkhov Date: Mon, 19 Oct 2026 22:32:23 -0700 Subject: [PATCH 080/101] build(sbom): keep a compiled component's nested licence texts A nested licence text was kept only when a copied wheel member came from its directory, so a component that reaches a wheel by being compiled in lost every bundled library's terms: jsoncpp in the OpenXR loader, hidapi and yuv2rgb in SDL3, spdlog's bundled fmt. Compiled members now record what their build read, from the compiler's dependency output (Makefile depfiles, the Ninja deps log), falling back to listed sources and include directories where a target has none. A nested text is kept when code under its directory reached the member. Nested rows are labelled as terms for part of the component, and a text several ids share is shown as a choice between them. Signed-off-by: Andrei Aristarkhov --- docs/source/references/sbom.rst | 27 +- scripts/sbom/sbom/build.py | 86 +-- scripts/sbom/sbom/discovery.py | 302 ++++++++++- scripts/sbom/sbom/document.py | 13 +- scripts/sbom/sbom/inventory.py | 63 ++- scripts/sbom/sbom/licensing.py | 104 +++- scripts/sbom/sbom/report.py | 8 + tests/python/sbom/conftest.py | 19 +- tests/python/sbom/synth.py | 158 +++++- tests/python/sbom/test_licensing.py | 58 +++ tests/python/sbom/test_nested_terms.py | 579 +++++++++++++++++++++ tests/python/sbom/test_spdx_conformance.py | 11 +- tests/python/sbom/test_static_archives.py | 36 +- 13 files changed, 1326 insertions(+), 138 deletions(-) create mode 100644 tests/python/sbom/test_nested_terms.py diff --git a/docs/source/references/sbom.rst b/docs/source/references/sbom.rst index 2b7b8fde1a..58dd6cb979 100644 --- a/docs/source/references/sbom.rst +++ b/docs/source/references/sbom.rst @@ -160,6 +160,13 @@ CMake's file API codemodel needs at least that, though the project builds without it. Reading the generator's own output instead would only work under Makefiles, since Ninja writes no ``link.txt``. +The compiler's dependency output + Which headers each object read: the per-object depfiles and ``compiler_depend.make`` + under the Makefile generators, ``ninja -t deps`` under Ninja — so a Ninja build is + collected where ``ninja`` is on ``PATH``, and the collector says so once when it is + not. Any target whose records do not cover every source it compiles falls back to + what it lists and includes. + Archives in the repository Every member of every tarball is hashed. A wheel member that matches one is recorded as having been unpacked from that archive. @@ -228,11 +235,21 @@ an exception's text ships too where the expression travels as a ``LicenseRef-``, document says so for every such text, in as many words, because a reader cannot tell a template from an author's own notice by looking at it. -A component can also state terms for one part of itself — a vendored asset set inside a -larger project, with its own ``LICENSE`` and its own holder. Those are read and packaged -too, but only where that directory reached the wheel, and they never change the component's -own licence: they name a holder for the bytes that shipped, which is what the notice -obligation is about. +A component can also state terms for one part of itself — a vendored asset set or a bundled +library inside a larger project, with its own ``LICENSE`` and its own holder. Those are read +and packaged too, but only where something under that directory reached the wheel: a file +copied or derived from it — a copy the build made into the component's own build tree +included — a source compiled from it, or a header the compiler's dependency output says it +read. Where a target compiles something its dependency output does not account for, the +sources it lists count instead, and so does any include directory above or below that +directory. An archive of the component linked in keeps all of them, since what it holds is +unknown. A REUSE ``.license`` sidecar covers only the file it annotates. Which of those kept +each text is in the build evidence. + +A bundled library compiled in ships its texts and holders, each listed as terms stated for +part of the component; the component's concluded licence stays its own grant. In the evidence +listed for a text, one that matches several identifiers sharing one reference text — +``GPL-3.0-only`` and ``GPL-3.0-or-later`` — is listed as matching either, not both. The distribution's own declared licence is an obligation like any other. A wheel declaring ``License-Expression`` and shipping no text for it stops the build, the same as a diff --git a/scripts/sbom/sbom/build.py b/scripts/sbom/sbom/build.py index a5df0ef11e..ef0d9e3301 100644 --- a/scripts/sbom/sbom/build.py +++ b/scripts/sbom/sbom/build.py @@ -24,7 +24,15 @@ from packaging.utils import canonicalize_name from . import discovery as discovery_module -from .discovery import project_copyright, project_supplier, sha256_file +from .discovery import ( + PREBUILT, + Reach, + covers, + project_copyright, + project_supplier, + reach_rank, + sha256_file, +) from .document import NOASSERTION from .inventory import Attribution from .wheelfile import Entry @@ -85,40 +93,53 @@ def default_license_data(build_dir: Path) -> Path: def _with_reached_nested(inventory, components: dict) -> dict: - """Drop nested terms whose directory contributed nothing to this wheel. + """Keep nested terms only where what they cover reached this wheel. - A nested LICENSE states a holder for part of a component -- the assets under - it -- and dropping it names the wrong holder for those bytes. Keeping all of - them names holders for code that never shipped. + A nested LICENSE states a holder for part of a component -- a bundled + library, an asset set -- and dropping it names the wrong holder for those + bytes. Keeping all of them names holders for code that never shipped. """ - reached: dict[str, set[str]] = {} - for attribution in inventory.attributions.values(): - if attribution.source_path: - for key in attribution.components: - reached.setdefault(key, set()).add(attribution.source_path) + # One entry per path and how, under its first member: members linking one + # library read the same headers, and every nested text scans these. + reached: dict[str, dict[tuple[str, str], Reach]] = {} + # A linked archive of the component holds what it holds: all of it reached. + prebuilt: dict[str, str] = {} + for member, attribution in inventory.attributions.items(): + for key, paths in attribution.read.items(): + into = reached.setdefault(key, {}) + for path, how in paths.items(): + known = into.get((path, how)) + if known is None or member < known.member: + into[(path, how)] = Reach(how, path, member) + for key, roles in attribution.components.items(): + if PREBUILT in roles: + prebuilt[key] = min(member, prebuilt.get(key, member)) + + def strength(hit: Reach): + return reach_rank(hit.how), hit.path, hit.member updated = {} for key, component in components.items(): - nested = [item for item in component.evidence if item.kind == "nested"] - if not nested: + if not any(item.kind == "nested" for item in component.evidence): continue - sources = reached.get(key, set()) - kept = [ - item - for item in nested - if any( - source.startswith(item.path.rsplit("/", 1)[0] + "/") - for source in sources - ) - ] - if len(kept) == len(nested): - continue - evidence = tuple( - item for item in component.evidence if item.kind != "nested" or item in kept - ) + evidence = [] + for item in component.evidence: + if item.kind != "nested": + evidence.append(item) + continue + hits = [ + hit + for hit in reached.get(key, {}).values() + if covers(hit.path, hit.how, item.scope) + ] + if key in prebuilt: + whole = component.root.name if component.root else key + hits.append(Reach(PREBUILT, whole, prebuilt[key])) + if hits: + evidence.append(replace(item, reached_by=min(hits, key=strength))) updated[key] = replace( component, - evidence=evidence, + evidence=tuple(evidence), copyright_text=discovery_module.notice_from(evidence), ) return updated @@ -292,11 +313,12 @@ def build( # component so the document, the reports, the packaged notices and the # evidence sidecar all describe the same thing. # - # Terms stated for one subdirectory apply only if that directory reached - # this wheel: a test suite's licence is not this wheel's business, a - # vendored asset set's is. And components carrying byte-identical terms - # share one LicenseRef -- settled here because deciding it in the renderer - # left the notices file naming an id the SBOM beside it did not define. + # Terms stated for one subdirectory apply only if something under it + # reached this wheel: a test suite's licence is not this wheel's business, + # a vendored asset set's or a compiled-in library's is. And components + # carrying byte-identical terms share one LicenseRef -- settled here because + # deciding it in the renderer left the notices file naming an id the SBOM + # beside it did not define. settled = _with_reached_nested(inventory, inventory.components_present) present = {**inventory.components_present, **settled} renamed = licensing.shared_refs(present.values()) diff --git a/scripts/sbom/sbom/discovery.py b/scripts/sbom/sbom/discovery.py index 349fdf03d8..92d08be8e1 100644 --- a/scripts/sbom/sbom/discovery.py +++ b/scripts/sbom/sbom/discovery.py @@ -21,6 +21,7 @@ import glob import hashlib import json +import logging import os import platform import re @@ -31,7 +32,8 @@ import zipfile from collections.abc import Iterable from dataclasses import dataclass, field, replace -from pathlib import Path +from pathlib import Path, PurePosixPath +from typing import NamedTuple from packageurl import PackageURL @@ -65,6 +67,8 @@ EMPTY_SHA256 = hashlib.sha256(b"").hexdigest() +_LOG = logging.getLogger("isaaccapture.sbom.discovery") + def component_of(display: str) -> str: """The component a `FileIndex` display path belongs to. @@ -294,6 +298,58 @@ class _Node: # library's own requirements land on its consumers' lines. linked: tuple[Path, ...] = () # outputs of targets in this graph inputs: tuple[LinkInput, ...] = () # everything else that can carry code + # The sources it compiles, as opposed to merely lists, and the directory + # its objects and their dependency output land in. + compiled: tuple[Path, ...] = () + objects: Path | None = None + + +# How a path of a component reached an artifact, strongest evidence first. A +# member made from a file of the component, then the sources compiled into it, +# then the headers the compiler's dependency output says it read. A target that +# compiles something but left no such output stands in the sources it lists and +# its include directories. A linked archive of the component is not a path: its +# contents are unknowable, so it stands for the whole component. +REACH = ( + "copied-file", + "derived-file", + "compiled-source", + "header-read", + "listed-source", + "include-dir", +) +PREBUILT = "prebuilt-library" + + +class Reach(NamedTuple): + """The evidence that kept a nested text: how, through which path, into + which wheel member.""" + + how: str + path: str + member: str + + +def reach_rank(how: str) -> int: + """Strongest first; a linked archive ranks below every path.""" + return REACH.index(how) if how in REACH else len(REACH) + + +def note_reach(found: dict, path, how: str) -> None: + """Record `how` for `path` unless stronger evidence is already there.""" + current = found.get(path) + if current is None or reach_rank(how) < reach_rank(current): + found[path] = how + + +def covers(path: str, how: str, scope: str) -> bool: + """Whether a reached path puts something under a nested text's scope in + the wheel. An include directory may have been read anywhere beneath it, + so one above the scope keeps it too.""" + reached, scope_path = PurePosixPath(path), PurePosixPath(scope) + return reached.is_relative_to(scope_path) or ( + how == "include-dir" and scope_path.is_relative_to(reached) + ) @dataclass(frozen=True) @@ -519,6 +575,18 @@ def _owner(path: Path, ownership: Ownership | None) -> str | None: return ownership.owner_of(path) if ownership is not None else None +def _include_owners(include: Path, ownership: Ownership | None) -> set[str]: + """The components an include directory credits. + + Which file in it was used is unknowable, so every component with files + there is credited, the same way a fetched include directory is. + """ + match = _DEPS_SOURCE.search(include.as_posix()) + if match: + return {match.group("name")} + return ownership.owners_under(include) if ownership is not None else set() + + class SupplierError(Exception): """The project does not name exactly one author to record as supplier.""" @@ -643,6 +711,132 @@ def _ask(*command: str) -> str: return result.stdout if result.returncode == 0 else "" +# A make rule as GCC and Clang write a depfile, and CMake's Makefile generators +# consolidate them into compiler_depend.make. +_MAKE_CONTINUATION = re.compile(r"\\\r?\n") +_MAKE_RULE = re.compile(r"^(?:\\.|[^:\\#])+:(?:\s+(?P.*))?$") +_MAKE_WORD = re.compile(r"(?:\\.|[^\s\\])+") +_MAKE_ESCAPE = re.compile(r"\\(.)") +_NINJA_DEPS = re.compile(r"^(?P\S.*): #deps \d+") +# The `CMakeFiles/.dir/` an output sits in, from the top of the build. +_OBJECTS_DIR = re.compile(r"^(?:.*?/)?CMakeFiles/[^/]+\.dir/") + + +def _make_rules(text: str) -> list[list[str]]: + """The prerequisites of each rule of a depfile that names any, unescaped. + + An object's rule names its source first; `-MP`'s rules for headers name + nothing. + """ + found = [] + for line in _MAKE_CONTINUATION.sub(" ", text).splitlines(): + match = _MAKE_RULE.match(line) + words = _MAKE_WORD.findall(match["prerequisites"] or "") if match else [] + if words: + found.append( + [_MAKE_ESCAPE.sub(r"\1", word).replace("$$", "$") for word in words] + ) + return found + + +def _ninja_deps(build_root: Path) -> dict[str, list[str]]: + """`ninja -t deps`: each output's recorded dependencies, by output path.""" + found: dict[str, list[str]] = {} + current: list[str] | None = None + for line in _ask("ninja", "-C", str(build_root), "-t", "deps").splitlines(): + match = _NINJA_DEPS.match(line) + if match: + current = found.setdefault(match["output"], []) + elif current is not None and line.startswith(" ") and line.strip(): + current.append(line.strip()) + return found + + +class DependencyLog: + """What the compiler recorded reading, per target objects directory.""" + + def __init__(self, generator: str, build_root: Path) -> None: + self._ninja_generator = "Ninja" in generator + self._build_root = build_root + # Ninja's records grouped by the objects directory they sit in. + self._ninja: dict[str, list[list[str]]] | None = None + self._resolved: dict[str, Path] = {} + self._read: dict[Path, tuple[set[Path], bool]] = {} + + def read(self, objects: Path, compiled: tuple[Path, ...]) -> tuple[set[Path], bool]: + """What compiling into `objects` read, and whether every source in + `compiled` has a record of its own: a stale object, or a generator + that keeps no record, leaves a gap, and a source another object + merely includes is not its own record.""" + if objects not in self._read: + read: set[Path] = set() + recorded: set[Path] = set() + wanted, own = set(compiled), objects.resolve() + for base, prerequisites in self._records(objects): + paths = [self._resolve_under(base, item) for item in prerequisites] + # A record for a source the target no longer compiles is stale; + # one CMake generated into the objects directory (a PCH) is + # current, except a unity source the codemodel no longer lists. + source = paths[0] + generated = source.is_relative_to(own) and not source.is_relative_to( + own / "Unity" + ) + if source not in wanted and not generated: + continue + recorded.add(source) + read.update(paths) + self._read[objects] = (read, set(compiled) <= recorded) + return self._read[objects] + + def _records(self, objects: Path) -> list[tuple[Path, list[str]]]: + """One record per object, its source first, with the directory its + relative paths are relative to.""" + if self._ninja_generator: + return self._ninja_records(objects) + return self._make_records(objects) if objects.is_dir() else [] + + def _ninja_records(self, objects: Path) -> list[tuple[Path, list[str]]]: + """Ninja's deps log names objects and paths from the top of the build.""" + if self._ninja is None: + self._ninja = {} + for output, items in _ninja_deps(self._build_root).items(): + match = _OBJECTS_DIR.match(output) + if match and items: + self._ninja.setdefault(match.group(0), []).append(items) + if not self._ninja: + _LOG.warning( + "`ninja -t deps` in %s recorded nothing; nested licence " + "texts are kept by what targets list and include instead. " + "Collect where `ninja` is on PATH.", + self._build_root, + ) + try: + prefix = objects.relative_to(self._build_root).as_posix() + "/" + except ValueError: + return [] + return [(self._build_root, items) for items in self._ninja.get(prefix, [])] + + def _make_records(self, objects: Path) -> list[tuple[Path, list[str]]]: + """A depfile is written where make runs the compiler, the target's own + build directory; compiler_depend.make is relative to the top.""" + files = [ + *((objects.parent.parent, path) for path in sorted(objects.rglob("*.d"))), + (self._build_root, objects / "compiler_depend.make"), + ] + return [ + (base, rule) + for base, path in files + if path.is_file() + for rule in _make_rules(path.read_text(encoding="utf-8", errors="replace")) + ] + + def _resolve_under(self, base: Path, item: str) -> Path: + joined = os.path.join(base, item) + if joined not in self._resolved: + self._resolved[joined] = Path(joined).resolve() + return self._resolved[joined] + + @functools.lru_cache(maxsize=None) def _compiler_search_dirs(compiler: str) -> list[Path] | None: """The `libraries:` list of ` -print-search-dirs`, GCC's and Clang's.""" @@ -689,6 +883,12 @@ def __init__(self, build_dir: Path, config: str | None = None) -> None: self._inspected: dict[ Path, tuple[str | None, int | None, list[str] | None] ] = {} + self._dependencies = DependencyLog("", build_dir) + self._linked: dict[str, tuple[_Node, ...]] = {} + # By (target name, which CMake keeps unique; the ownership asked with). + self._node_reach: dict[ + tuple[str, Ownership | None], dict[str, dict[Path, str]] + ] = {} self._load(config) def _load(self, config: str | None) -> None: @@ -709,6 +909,7 @@ def _load(self, config: str | None) -> None: codemodel = json.loads(model_file.read_text(encoding="utf-8")) source_root = Path(codemodel["paths"]["source"]) build_root = Path(codemodel["paths"]["build"]) + self._dependencies = DependencyLog(generator, build_root) configurations = codemodel["configurations"] chosen = next( @@ -742,11 +943,8 @@ def resolve(raw: str, root: Path) -> Path: for group in target.get("compileGroups", []) for item in group.get("includes", []) } - link_dir = ( - resolve(target.get("paths", {}).get("build", "."), build_root) - if per_directory - else build_root - ) + target_dir = resolve(target.get("paths", {}).get("build", "."), build_root) + link_dir = target_dir if per_directory else build_root link = target.get("link", {}) words, unparsed = _linker_words(link.get("commandFragments", [])) found = search.get(link.get("language", "")) @@ -767,6 +965,13 @@ def resolve(raw: str, root: Path) -> Path: dependencies=tuple( item["id"] for item in target.get("dependencies", []) ), + compiled=tuple( + resolve(item["path"], source_root).resolve() + for item in target.get("sources", []) + if "compileGroupIndex" in item + ), + # Where the Makefile and Ninja generators put a target's objects. + objects=target_dir / "CMakeFiles" / f"{target['name']}.dir", ) for item in target.get("artifacts", []): output = resolve(item["path"], build_root).resolve() @@ -955,15 +1160,21 @@ def static_inputs(self, artifact: Artifact) -> dict[LinkInput, set[str]]: """ if artifact.target_id not in self._static: found: dict[LinkInput, set[str]] = {} - for node in self._reach( - artifact.target_id, - lambda node: node.kind in ("EXECUTABLE", "MODULE_LIBRARY"), - ): + for node in self._linked_closure(artifact.target_id): for item in node.inputs: found.setdefault(item, set()).add(node.name) self._static[artifact.target_id] = found return self._static[artifact.target_id] + def _linked_closure(self, target_id: str) -> tuple[_Node, ...]: + """`_reach` without other executables and modules, nor what only they + reach: what can end up inside the artifact.""" + if target_id not in self._linked: + self._linked[target_id] = self._reach( + target_id, lambda node: node.kind in ("EXECUTABLE", "MODULE_LIBRARY") + ) + return self._linked[target_id] + def contributions( self, artifact: Artifact, ownership: Ownership | None = None ) -> dict[str, set[str]]: @@ -971,7 +1182,8 @@ def contributions( Walks the target's dependency closure. Compiled sources name a component outright; include paths are the only trace a header-only dependency - leaves anywhere; a linked archive or object names whoever holds it. + leaves in the codemodel; a linked archive or object names whoever holds + it. """ found: dict[str, set[str]] = {} @@ -984,19 +1196,54 @@ def note(key: str, how: str) -> None: if owner: note(owner, "compiled-source") for include in node.includes: - match = _DEPS_SOURCE.search(include.as_posix()) - if match: - note(match.group("name"), "header-include") - elif ownership is not None: - # Which file in the directory was used is unknowable, so - # every component with files there is credited, the same - # way a fetched include directory is. - for owner in ownership.owners_under(include): - note(owner, "header-include") + for owner in _include_owners(include, ownership): + note(owner, "header-include") for item in self.static_inputs(artifact): owner = owned_input(item, ownership) if owner: - note(owner, "prebuilt-library") + note(owner, PREBUILT) + return found + + def reached( + self, artifact: Artifact, ownership: Ownership | None = None + ) -> dict[str, dict[Path, str]]: + """The resolved paths of each component that went into an artifact, + each with its strongest `REACH`.""" + reached: dict[str, dict[Path, str]] = {} + for node in self._linked_closure(artifact.target_id): + for key, paths in self._reach_of(node, ownership).items(): + into = reached.setdefault(key, {}) + for path, how in paths.items(): + note_reach(into, path, how) + return reached + + def _reach_of( + self, node: _Node, ownership: Ownership | None + ) -> dict[str, dict[Path, str]]: + """One target's owned paths by `REACH`; what it lists and includes only + where it compiles something its dependency output does not cover.""" + if (node.name, ownership) in self._node_reach: + return self._node_reach[(node.name, ownership)] + found: dict[str, dict[Path, str]] = {} + + def mark(owner: str | None, path: Path, how: str) -> None: + if owner: + note_reach(found.setdefault(owner, {}), path, how) + + if node.objects is None: + return found + for source in node.compiled: + mark(_owner(source, ownership), source, "compiled-source") + read, covered = self._dependencies.read(node.objects, node.compiled) + for path in read: + mark(_owner(path, ownership), path, "header-read") + if not covered: + for source in node.sources: + mark(_owner(source, ownership), source, "listed-source") + for include in node.includes: + for owner in _include_owners(include, ownership): + mark(owner, include, "include-dir") + self._node_reach[(node.name, ownership)] = found return found def first_party( @@ -1035,9 +1282,16 @@ def __init__(self) -> None: self._roots: list[tuple[Path, str]] = [] # directory -> (owners credited outright, owners credited if alone) self._index: dict[Path, tuple[set[str], set[str]]] | None = None + # Every header a compile read is asked about, most of them many times. + self._owners: dict[Path, str | None] = {} def register_root(self, root: Path, key: str) -> None: self._roots.append((root.resolve(), key)) + self._owners.clear() + + def roots_of(self, key: str) -> list[Path]: + """The resolved trees registered for a component.""" + return [root for root, owner in self._roots if owner == key] def register_file(self, path: Path, key: str, *, by_directory: bool = True) -> None: """Record a file's owner. @@ -1048,8 +1302,14 @@ def register_file(self, path: Path, key: str, *, by_directory: bool = True) -> N """ self._files[path.resolve()] = (key, by_directory) self._index = None + self._owners.clear() def owner_of(self, path: Path) -> str | None: + if path not in self._owners: + self._owners[path] = self._find_owner(path) + return self._owners[path] + + def _find_owner(self, path: Path) -> str | None: resolved = path.resolve() entry = self._files.get(resolved) if entry is not None: diff --git a/scripts/sbom/sbom/document.py b/scripts/sbom/sbom/document.py index 12b56fde92..af60c30fdd 100644 --- a/scripts/sbom/sbom/document.py +++ b/scripts/sbom/sbom/document.py @@ -44,6 +44,11 @@ def verification_code(sha1_digests: list[str]) -> str: return hashlib.sha1("".join(sorted(sha1_digests)).encode("ascii")).hexdigest() # noqa: S324 +def _scored(match) -> str: + license_id, containment, coverage = match + return f"{license_id} (containment {containment:.2f}, coverage {coverage:.2f})" + + def _attribution_texts(component: Component, dist_info: str) -> list[str]: texts = [ f"License evidence: {item.path} (sha256:{item.sha256}, {item.size} bytes, " @@ -51,10 +56,9 @@ def _attribution_texts(component: Component, dist_info: str) -> list[str]: # Whitespace after the path: validate.py reads it up to the first one. f"{report.packaged_license_path(dist_info, component.key, item)}" + ( - " identified as " - + "; ".join( - f"{license_id} (containment {containment:.2f}, coverage {coverage:.2f})" - for license_id, containment, coverage in item.matches + " " + + licensing.matched_prose( + item.matches, name=_scored, join="; ", lead="identified as " ) if item.matches # The SPDX reference text for an identifier the component's files @@ -64,6 +68,7 @@ def _attribution_texts(component: Component, dist_info: str) -> list[str]: if item.origin == licensing.SPDX_REFERENCE else "" ) + + (f" — {report.NESTED_ROLE}" if item.kind == "nested" else "") for item in component.evidence ] if not any(item.kind == "grant" for item in component.evidence): diff --git a/scripts/sbom/sbom/inventory.py b/scripts/sbom/sbom/inventory.py index 155a84d400..ef6aa44600 100644 --- a/scripts/sbom/sbom/inventory.py +++ b/scripts/sbom/sbom/inventory.py @@ -22,6 +22,7 @@ Component, EMPTY_SHA256, LinkInput, + note_reach, component_of, owned_input, resolve_static_archive, @@ -76,10 +77,13 @@ class Attribution: # the other made the same fact answerable two ways depending on which index # happened to hold the bytes. exact: bool = False - # Where in the component these bytes came from, where an index named it. - # The detail string says it in prose; this is the same fact in a form the - # rest of the collector can use to ask what else sits in that directory. + # Where in the component these bytes came from, where an index named it: + # the detail string's fact in a form the evidence JSON can carry. source_path: str | None = None + # Each component's paths these bytes were made from -- the file copied, or + # what compiling it read -- named as `source_path` names them, with their + # `discovery.REACH`. + read: dict[str, dict[str, str]] = field(default_factory=dict) def as_json(self) -> dict: return { @@ -200,8 +204,59 @@ def _from_artifact(self, artifact, path: str, detail: str) -> Attribution: detail=f"{detail}; linked by target {artifact.target}", components={key: set(how) for key, how in contributions.items()}, primary=primary, + read=self._displayed(self.graph.reached(artifact, self.ownership)), ) + def _displayed( + self, reached: dict[str, dict[Path, str]] + ) -> dict[str, dict[str, str]]: + """Reached paths as a component's own files are named: `/`. + + Only a component with a tree of its own has a name for them, and only + such a component states terms for part of itself. A file the build + copied or generated into another tree registered for the component -- + its `-build` tree -- stands where it sits relative to that tree, and is + also any file of the source tree holding the same bytes. An include + directory holding a whole tree stands for the component's root. + """ + shown: dict[str, dict[str, str]] = {} + for key, paths in reached.items(): + component = self.components.get(key) + if component is None or component.root is None: + continue + name = component.root.name + source = component.root.resolve() + trees = [ + source, + *(root for root in self.ownership.roots_of(key) if root != source), + ] + into = shown.setdefault(key, {}) + for path, how in paths.items(): + tree = next((root for root in trees if path.is_relative_to(root)), None) + if tree is None: + if how == "include-dir" and any( + root.is_relative_to(path) for root in trees + ): + note_reach(into, name, how) + continue + for display in self._names_in(name, tree != source, tree, path): + note_reach(into, display, how) + return shown + + def _names_in(self, name: str, built: bool, tree: Path, path: Path) -> list[str]: + """`path` as named in the source tree `name`; see `_displayed`.""" + positional = (Path(name) / path.relative_to(tree)).as_posix() + digest = self.build_files.digest_of(path) if built else None + # An empty file matches every other one, which says nothing. + if digest is None or digest == EMPTY_SHA256: + return [positional] + same = [ + display + for display in self.source_files.all_by_hash.get(digest, []) + if display.startswith(name + "/") + ] + return [*same, positional] + def _static_input(self, item: LinkInput, targets: set[str]) -> str | None: """The component a linked archive or object belongs to, if any.""" record = self.static_inputs.get(item.key) @@ -373,6 +428,7 @@ def _trace( components={key: {"copied-file"}}, primary=key, source_path=component_file, + read={key: {component_file: "copied-file"}}, ) member = self.archives.by_hash.get(candidate_digest) @@ -678,6 +734,7 @@ def _resolve_by_path( components={key: {"copied-file" if same else "derived-file"}}, primary=key, source_path=component_file, + read={key: {component_file: "copied-file" if same else "derived-file"}}, ) build_file = self.build_files.path_suffix_match(wheel_path) diff --git a/scripts/sbom/sbom/licensing.py b/scripts/sbom/sbom/licensing.py index a1dc513f54..1d14803387 100644 --- a/scripts/sbom/sbom/licensing.py +++ b/scripts/sbom/sbom/licensing.py @@ -14,7 +14,8 @@ import json import re from functools import lru_cache -from dataclasses import dataclass +from dataclasses import dataclass, replace +from typing import TYPE_CHECKING from license_expression import ( ExpressionError, @@ -24,6 +25,9 @@ ) from pathlib import Path +if TYPE_CHECKING: # pragma: no cover + from .discovery import Reach + # Root-level names that carry a grant, and the ones that carry notices instead. # Both are packaged; only the first kind is matched against the corpus. @@ -85,12 +89,18 @@ class LicenseEvidence: # "pool" a REUSE LICENSES/ entry that applies to individual files only, and # "nested" terms stated for one subdirectory of a larger component. kind: str + # Display text for a reader, not an expression; the ids are in `matches`. identified: str | None = None matches: tuple[tuple[str, float, float], ...] = () spdx_tag: str | None = None + # For "nested" terms: what they cover, as a path in the component -- their + # directory, or the one file a REUSE `.license` sidecar annotates -- and, + # where a wheel keeps them, the strongest evidence that it reached a member. + scope: str | None = None + reached_by: Reach | None = None def as_json(self) -> dict: - return { + found = { "component": self.component, "origin": self.origin, "path": self.path, @@ -108,6 +118,9 @@ def as_json(self) -> dict: ], "spdx_tag": self.spdx_tag, } + if self.reached_by: + found["reached_by"] = self.reached_by._asdict() + return found # What counts as a text that discharges the packaging obligation. A REUSE pool @@ -368,14 +381,64 @@ def _dominated( for right in scored: if left == right or right in dropped: continue - shared = len(entries[left].grams & entries[right].grams) - related = shared / min(entries[left].size, entries[right].size) - if related >= _FAMILY_THRESHOLD and scored[right][2] > scored[left][2]: + if _family(entries, left, right) and scored[right][2] > scored[left][2]: dropped.add(left) break return dropped +def _family(entries: dict[str, Reference], left: str, right: str) -> bool: + if left not in entries or right not in entries: + return False + shared = len(entries[left].grams & entries[right].grams) + return shared / min(entries[left].size, entries[right].size) >= _FAMILY_THRESHOLD + + +def alternatives(matches) -> list[tuple[str, ...]]: + """Matched ids, grouped where one text cannot tell them apart. + + Family members that tie survive `_dominated` together: GPL-3.0-only and + GPL-3.0-or-later share one reference text, and which applies is a notice's + choice, not the text's. + """ + entries = corpus()[1] + groups: list[list[str]] = [] + for license_id, _, _ in matches: + group = next( + ( + group + for group in groups + if any(_family(entries, license_id, other) for other in group) + ), + None, + ) + if group is None: + groups.append([license_id]) + else: + group.append(license_id) + return [tuple(group) for group in groups] + + +def matched_prose(matches, name=lambda match: match[0], join=" AND ", lead=""): + """What a text matched, for a reader: a tie is one text matching either + reference, never both licences at once. `name` renders one match, and + `lead` precedes the matches named outright, joined by `join`; a prose + clause is set off with "; " so no " AND " ever reads as joining it.""" + if not matches: + return None + by_id = {match[0]: match for match in matches} + groups = alternatives(matches) + single = [name(by_id[group[0]]) for group in groups if len(group) == 1] + prose = [ + "matches the reference text of " + + " or ".join(name(by_id[item]) for item in group) + for group in groups + if len(group) > 1 + ] + parts = ([lead + join.join(single)] if single else []) + prose + return ("; " if prose else join).join(parts) + + def _read(path: Path) -> str | None: try: data = path.read_bytes() @@ -394,11 +457,13 @@ def evidence_for( ) -> LicenseEvidence: # A reference text is not matched: scoring it against the corpus it came # from returns a perfect score, which reads as a measurement of the - # component's own file and is a measurement of nothing. - matchable = kind == "grant" and origin != SPDX_REFERENCE + # component's own file and is a measurement of nothing. Nested terms are + # matched to tell their holders from the licence's steward, never to + # conclude the component's licence. + matchable = kind in ("grant", "nested") and origin != SPDX_REFERENCE tag = _SPDX_TAG.search(text[:4096]) if matchable else None matches = tuple(identify(text)) if matchable else () - identified = " AND ".join(license_id for license_id, _, _ in matches) or None + identified = matched_prose(matches) if origin == SPDX_REFERENCE: # Named by the identifier it was fetched for, with no score behind it. identified = Path(display).stem @@ -444,7 +509,8 @@ def discover_in_tree( # A subdirectory can carry its own terms and its own holder -- a vendored # asset set inside a larger project. Collected as "nested" so it neither # discharges the component's packaging obligation nor changes its licence, - # and kept only where that directory reached the wheel; see `nested_for`. + # and kept only where what it covers reached the wheel; see + # `build._with_reached_nested`. for path in sorted(root.rglob("*")) if root.is_dir() else []: if ( path.is_file() @@ -459,7 +525,14 @@ def discover_in_tree( if text is None: continue display = f"{display_prefix}/{path.relative_to(root).as_posix()}" - found.append(evidence_for(component, "component-file", display, text, kind)) + item = evidence_for(component, "component-file", display, text, kind) + if kind == "nested": + annotated = path.name.removesuffix(".license") + sidecar = path.with_name(annotated) if annotated else path + covered = sidecar if sidecar != path and sidecar.is_file() else path.parent + scope = f"{display_prefix}/{covered.relative_to(root).as_posix()}" + item = replace(item, scope=scope) + found.append(item) # "nested" never satisfies the grant test below; that is the point. if not any(item.kind == "grant" for item in found): @@ -832,14 +905,21 @@ def notice_is_the_licence_authors(notice: str, identified: list[str]) -> bool: keeping a list of stewards. """ entries = corpus()[1] - collapsed = " ".join(notice.split()) + collapsed = _steward_form(notice) return any( (reference := entries.get(license_id)) - and collapsed in " ".join(reference.text.split()) + and collapsed in _steward_form(reference.text) for license_id in identified ) +def _steward_form(text: str) -> str: + """Folded the way copies of one steward's line differ: the GPL's reads + `(C)` and `http://` as upstreams ship it, `©` and `https://` in the list.""" + folded = " ".join(text.split()).lower().replace("©", "(c)") + return folded.replace("https://", "http://") + + def read_notices(text: str) -> list[str]: """Every concrete copyright notice a licence text states. diff --git a/scripts/sbom/sbom/report.py b/scripts/sbom/sbom/report.py index dc7b20979a..0ed2df77b0 100644 --- a/scripts/sbom/sbom/report.py +++ b/scripts/sbom/sbom/report.py @@ -80,6 +80,11 @@ def declared_license_path(dist_info: str, member: str) -> str: return member.removeprefix(licenses_root(dist_info)) +# What a nested text is, wherever one is listed. Some restate the component's +# own grant, so this says only where they apply. +NESTED_ROLE = "terms stated for part of this component" + + def packaged_license_path(dist_info: str, component_key: str, evidence) -> str: """Where one component's license text lands inside the wheel. @@ -174,6 +179,8 @@ def notices_markdown( ) + ", not this component's own file" if item.origin == licensing.SPDX_REFERENCE + else f" — {NESTED_ROLE}" + if item.kind == "nested" else "" ) ) @@ -301,6 +308,7 @@ def report_markdown(payload: dict) -> str: parts = [ f"`{record['path']}`" + (f" → {record['identified']}" if record["identified"] else "") + + (f" ({NESTED_ROLE})" if record["kind"] == "nested" else "") + ( " (SPDX reference text, substituted)" if record["origin"] == licensing.SPDX_REFERENCE diff --git a/tests/python/sbom/conftest.py b/tests/python/sbom/conftest.py index 907744a2b1..8cc7bc4eb4 100644 --- a/tests/python/sbom/conftest.py +++ b/tests/python/sbom/conftest.py @@ -11,7 +11,6 @@ import pytest import synth -from sbom import build as build_module from sbom import licensing @@ -102,23 +101,21 @@ def workspace(tmp_path): @pytest.fixture def built(workspace): """The workspace after the collector has packaged evidence into the wheel.""" - out_dir = workspace.root / "sbom" - manifest = build_module.build( - workspace.root, workspace.build, workspace.wheel, out_dir - ) + collected = synth.collect(workspace) + out_dir = collected["out_dir"] manifest_path = out_dir / "manifest.json" manifest_path.write_text( - json.dumps(manifest, indent=2, sort_keys=True) + "\n", encoding="utf-8" + json.dumps(collected["manifest"], indent=2, sort_keys=True) + "\n", + encoding="utf-8", ) - stem = synth.WHEEL_NAME.removesuffix(".whl") return { "workspace": workspace, "root": workspace.root, "wheel": workspace.wheel, "out_dir": out_dir, - "manifest": manifest, + "manifest": collected["manifest"], "manifest_path": manifest_path, - "evidence_path": out_dir / f"{stem}.build-evidence.json", - "spdx_path": out_dir / f"{stem}.spdx.json", - "report_path": out_dir / f"{stem}.licenses.json", + "evidence_path": out_dir / f"{synth.STEM}.build-evidence.json", + "spdx_path": out_dir / f"{synth.STEM}.spdx.json", + "report_path": out_dir / f"{synth.STEM}.licenses.json", } diff --git a/tests/python/sbom/synth.py b/tests/python/sbom/synth.py index 0802772419..6a7d0851e4 100644 --- a/tests/python/sbom/synth.py +++ b/tests/python/sbom/synth.py @@ -32,8 +32,11 @@ import zipfile from pathlib import Path +from sbom import build as build_module + CONFIG = "Release" WHEEL_NAME = "isaaccapture-0.4.0-cp312-cp312-manylinux_2_35_x86_64.whl" +STEM = WHEEL_NAME.removesuffix(".whl") DIST_INFO = "isaaccapture-0.4.0.dist-info" SDK_ARCHIVE = "deps/vendor/demosdk-1.2.0-linux-amd64.tar.gz" SDK_DEMO_BUILD_ID = "1111111111111111111111111111111111111111" @@ -163,6 +166,8 @@ def phdr(index: int, kind: int, offset: int, size: int) -> None: "BSL-1.0", "CC-BY-4.0", "ECL-2.0", + "GPL-3.0-only", + "GPL-3.0-or-later", "MIT", "MIT-0", "Zlib", @@ -550,24 +555,13 @@ def _file_api_reply(root: Path, build: Path, extension: str) -> None: }, ] - reply = build / ".cmake" / "api" / "v1" / "reply" + reply = _reply(build) index = [] for spec in targets: - name = spec["name"] - payload = { - "name": name, - "type": spec["type"], - "artifacts": [{"path": item} for item in spec["artifacts"]], - "sources": [{"path": item} for item in spec["sources"]], - "compileGroups": [ - {"includes": [{"path": item} for item in spec["includes"]]} - ], - "dependencies": [{"id": dep} for dep in spec["dependencies"]], - } - write(reply / f"target-{name}-Release.json", json.dumps(payload, indent=1)) - index.append( - {"id": name, "name": name, "jsonFile": f"target-{name}-Release.json"} - ) + entry = _target_entry(spec["name"]) + payload = _target_payload(**spec) + write(reply / entry["jsonFile"], json.dumps(payload, indent=1)) + index.append(entry) codemodel = { "kind": "codemodel", @@ -594,6 +588,38 @@ def _file_api_reply(root: Path, build: Path, extension: str) -> None: ) +def _source(path: str) -> dict: + """A codemodel source entry; one CMake compiles names its compile group.""" + compiled = path.endswith((".c", ".cpp")) + return {"path": path, **({"compileGroupIndex": 0} if compiled else {})} + + +def _target_payload( + name: str, + type: str, # noqa: A002 - the codemodel's own key + artifacts: list[str], + sources: list[str], + includes: list[str] | None, + dependencies: list[str], +) -> dict: + """A codemodel target; one with no compile groups compiles nothing.""" + payload = { + "name": name, + "type": type, + "artifacts": [{"path": item} for item in artifacts], + "sources": [_source(item) for item in sources], + "dependencies": [{"id": dep} for dep in dependencies], + } + if includes is not None: + payload["compileGroups"] = [{"includes": [{"path": item} for item in includes]}] + return payload + + +def _target_entry(name: str) -> dict: + """A target's entry in the codemodel's configuration.""" + return {"id": name, "name": name, "jsonFile": f"target-{name}-{CONFIG}.json"} + + @contextlib.contextmanager def _edit_json(path: Path): """Read a reply file, let the caller change it, write it back.""" @@ -643,8 +669,8 @@ def link_line( if sysroot is not None: payload["link"]["sysroot"] = {"path": str(sysroot)} payload["dependencies"] += [{"id": item} for item in dependencies] - if directory is not None: - payload["paths"] = {"build": directory, "source": directory} + if directory is not None: + set_directory(build, target, directory) def set_toolchains( @@ -692,6 +718,82 @@ def add_include(build: Path, target: str, directory: Path) -> None: payload["compileGroups"][0]["includes"].append({"path": str(directory)}) +def add_source(build: Path, target: str, path: Path) -> None: + """Compile one more source into a fixture target.""" + with _edit_target(build, target) as payload: + payload["sources"].append(_source(str(path))) + + +def depfile( + build: Path, + target: str, + source: Path | str, + read: tuple[Path | str, ...] = (), + directory: str = ".", +) -> None: + """The depfile GCC leaves beside one of a target's objects under the + Makefile generators: the object, its source, every header it read, and + `-MP`'s empty rule per header. `directory` is the target's build directory. + """ + obj = f"CMakeFiles/{target}.dir/{Path(source).name}.o" + rule = " \\\n ".join([f"{obj}: {source}", *(str(item) for item in read)]) + write( + build / directory / f"{obj}.d", + rule + "\n" + "".join(f"\n{item}:\n" for item in read), + ) + + +def compiler_depend( + build: Path, target: str, rules: dict[str, tuple[str, ...]], directory: str = "." +) -> None: + """compiler_depend.make as CMake consolidates a target's depfiles: one rule + per object, paths inside the build tree relative to its top. No rules is + the placeholder CMake writes before the first build.""" + path = build / directory / "CMakeFiles" / f"{target}.dir" / "compiler_depend.make" + if not rules: + write( + path, + f"# Empty compiler generated dependencies file for {target}.\n" + "# This may be replaced when dependencies are built.\n", + ) + return + write( + path, + "# CMAKE generated file: DO NOT EDIT!\n\n" + + "\n".join( + " \\\n ".join([f"{obj}: {prerequisites[0]}", *prerequisites[1:]]) + for obj, prerequisites in rules.items() + ) + + "\n", + ) + + +def add_target( + build: Path, name: str, kind: str, sources: tuple[str, ...], dependent: str +) -> None: + """One more target, which `dependent` depends on, producing nothing shipped.""" + entry = _target_entry(name) + payload = _target_payload( + name=name, + type=kind, + artifacts=[f"tools/{name}"] if kind == "EXECUTABLE" else [], + sources=list(sources), + includes=None if kind == "UTILITY" else [], + dependencies=[], + ) + write(_reply(build) / entry["jsonFile"], json.dumps(payload)) + with _edit_json(_reply(build) / "codemodel-v2-fixture.json") as codemodel: + codemodel["configurations"][0]["targets"].append(entry) + with _edit_target(build, dependent) as target: + target["dependencies"].append({"id": name}) + + +def set_directory(build: Path, target: str, directory: str) -> None: + """Put a fixture target in a subdirectory of the build tree.""" + with _edit_target(build, target) as payload: + payload["paths"] = {"build": directory, "source": directory} + + def set_type(build: Path, target: str, kind: str) -> None: """Change a fixture target's CMake type.""" with _edit_target(build, target) as payload: @@ -883,6 +985,26 @@ def wheel_payload(workspace: Workspace) -> dict[str, bytes]: return payload +def collect(workspace: Workspace, **options) -> dict: + """Run the collector over the workspace's wheel and read back what it wrote.""" + out_dir = workspace.root / "sbom" + manifest = build_module.build( + workspace.root, workspace.build, workspace.wheel, out_dir, **options + ) + with zipfile.ZipFile(workspace.wheel) as archive: + notices = archive.read(f"{DIST_INFO}/licenses/THIRD-PARTY-NOTICES.md") + return { + "out_dir": out_dir, + "manifest": manifest, + **{ + kind: json.loads((out_dir / f"{STEM}.{kind}.json").read_text("utf-8")) + for kind in ("spdx", "licenses", "build-evidence") + }, + "md": (out_dir / f"{STEM}.licenses.md").read_text("utf-8"), + "notices": notices.decode("utf-8"), + } + + def _record_hash(data: bytes) -> str: digest = hashlib.sha256(data).digest() return "sha256=" + base64.urlsafe_b64encode(digest).rstrip(b"=").decode("ascii") diff --git a/tests/python/sbom/test_licensing.py b/tests/python/sbom/test_licensing.py index a4de013648..0e256f641c 100644 --- a/tests/python/sbom/test_licensing.py +++ b/tests/python/sbom/test_licensing.py @@ -7,6 +7,7 @@ import pytest import synth +from sbom import discovery, document from sbom import licensing @@ -246,6 +247,63 @@ def test_a_component_carrying_two_licenses_combines_them(): assert concluded == "Apache-2.0 AND CC-BY-4.0" +def test_a_bare_license_file_covers_its_directory(tmp_path): + synth.write(tmp_path / "LICENSE", synth.spdx_text("MIT")) + synth.write(tmp_path / "third/.license", "SPDX-FileCopyrightText: 2020 A\n") + # Named like a sidecar, with no file beside it to annotate. + synth.write(tmp_path / "third/stb.license", synth.spdx_text("MIT")) + + nested = [ + item + for item in licensing.discover_in_tree("thing", tmp_path, "thing-src") + if item.kind == "nested" + ] + + assert [(item.path, item.scope) for item in nested] == [ + ("thing-src/third/.license", "thing-src/third"), + ("thing-src/third/stb.license", "thing-src/third"), + ] + + +def test_a_text_carrying_two_licences_lists_both_not_a_choice(): + dual = licensing.evidence_for( + "dual", + "component-file", + "dual/LICENSE", + synth.spdx_text("Apache-2.0") + "\n\n" + synth.spdx_text("CC-BY-4.0"), + "grant", + ) + + assert dual.identified == "Apache-2.0 AND CC-BY-4.0" + component = discovery.Component( + key="dual", + name="dual", + kind="source-tree", + supplier="NOASSERTION", + homepage="NOASSERTION", + purl="pkg:generic/dual", + version="1", + download_location="NOASSERTION", + source_info="", + evidence=(dual,), + ) + rendered = document._attribution_texts(component, synth.DIST_INFO)[0] + assert " identified as Apache-2.0 (" in rendered + assert "; CC-BY-4.0 (" in rendered + assert "reference text of" not in rendered + + +def test_a_choice_is_never_joined_to_a_licence_with_and(): + gpl = [("GPL-3.0-only", 1.0, 0.6), ("GPL-3.0-or-later", 1.0, 0.6)] + alternatives = "matches the reference text of GPL-3.0-only or GPL-3.0-or-later" + + assert licensing.matched_prose(gpl) == alternatives + assert ( + licensing.matched_prose([("Apache-2.0", 1.0, 0.2), *gpl, ("MIT", 1.0, 0.1)]) + == f"Apache-2.0 AND MIT; {alternatives}" + ) + + def test_an_index_of_licenses_is_not_a_grant(tmp_path): """A cone-mode sparse checkout keeps root files, so a repository that merely catalogues licences can present names that look like grants.""" diff --git a/tests/python/sbom/test_nested_terms.py b/tests/python/sbom/test_nested_terms.py new file mode 100644 index 0000000000..689e5bf2a6 --- /dev/null +++ b/tests/python/sbom/test_nested_terms.py @@ -0,0 +1,579 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Terms a component states for one part of itself ship only where that part did.""" + +from __future__ import annotations + +import json +import logging +from types import SimpleNamespace + +import pytest +import synth +from sbom import build as build_module +from sbom import discovery +from sbom import evidence as evidence_module +from sbom import inventory as inventory_module +from sbom import licensing +from sbom import report + +BUNDLED = "Copyright (c) 2019 Bundled Authors\n\n" + "Permission is hereby granted." +EXTENSION = "isaaccapture/_ext.cpython-312-x86_64-linux-gnu.so" + + +@pytest.fixture +def alpha(workspace): + return workspace.build / "_deps" / "alpha-src" + + +@pytest.fixture +def beta(workspace): + return workspace.build / "_deps" / "beta-src" + + +def _deps(workspace): + return workspace.build / "_deps" + + +def _compiled(workspace): + """The sources the fixture's `ext` target compiles, as a depfile names them.""" + return ( + (workspace.root / "src/cpp/module.cpp").resolve(), + (_deps(workspace) / "alpha-src/alpha.cpp").resolve(), + ) + + +def _depfiles(workspace, read=(), extra=()) -> None: + """A depfile per object `ext` compiles; the first also read `read`.""" + first, *rest = (*_compiled(workspace), *(item.resolve() for item in extra)) + synth.depfile(workspace.build, "ext", first, tuple(read)) + for source in rest: + synth.depfile(workspace.build, "ext", source) + + +def _kept(workspace, built=None) -> dict[str, dict | None]: + """Each nested text the build found, with what kept it, or None if dropped.""" + built = built or synth.collect(workspace) + found = { + item["path"]: item.get("reached_by") + for component in built["build-evidence"]["components"].values() + for item in component["evidence"] + if item["kind"] == "nested" + } + # What was dropped is absent; name it so a test can say so. + for root in sorted(_deps(workspace).glob("*-src")): + for item in licensing.discover_in_tree(root.name, root, root.name): + if item.kind == "nested": + found.setdefault(item.path, None) + return found + + +def _package(built, name): + return next(item for item in built["spdx"]["packages"] if item["name"] == name) + + +def test_a_compiled_in_library_keeps_its_terms_and_holder(workspace, alpha): + synth.write(alpha / "third/bundled.cpp", "// bundled\n") + synth.write(alpha / "third/LICENSE", BUNDLED) + synth.write(alpha / "test/LICENSE", "Copyright (c) 2018 Test Authors\n") + synth.add_source(workspace.build, "ext", alpha / "third/bundled.cpp") + _depfiles(workspace, extra=(alpha / "third/bundled.cpp",)) + + built = synth.collect(workspace) + kept = _kept(workspace, built) + + assert kept["alpha-src/third/LICENSE"]["how"] == "compiled-source" + assert kept["alpha-src/third/LICENSE"]["path"] == "alpha-src/third/bundled.cpp" + assert kept["alpha-src/test/LICENSE"] is None + copyright_text = _package(built, "alpha")["copyrightText"] + assert "Bundled Authors" in copyright_text + assert "Test Authors" not in copyright_text + + +def test_bundled_gpl_text_names_no_steward_and_reads_as_alternatives(workspace, alpha): + # As upstreams ship it: `(C)` and `http://`, where the list has `©` and `https://`. + gpl = ( + synth.spdx_text("GPL-3.0-only") + .replace("©", "(C)") + .replace("https://fsf.org/", "http://fsf.org/") + ) + synth.write(alpha / "third/bundled.cpp", "// bundled\n") + synth.write(alpha / "third/LICENSE-gpl3.txt", gpl) + synth.add_source(workspace.build, "ext", alpha / "third/bundled.cpp") + _depfiles(workspace, extra=(alpha / "third/bundled.cpp",)) + + built = synth.collect(workspace) + + alternatives = "matches the reference text of GPL-3.0-only or GPL-3.0-or-later" + package = _package(built, "alpha") + assert "Free Software Foundation" not in package["copyrightText"] + assert package["licenseConcluded"] == "MIT" + texts = [item for item in package["attributionTexts"] if "LICENSE-gpl3" in item] + assert "matches the reference text of GPL-3.0-only (" in texts[0] + assert texts[0].endswith(report.NESTED_ROLE) + row = next(line for line in built["md"].splitlines() if "| alpha |" in line) + assert f"→ {alternatives} ({report.NESTED_ROLE})" in row + assert "GPL-3.0-only AND" not in built["md"] + json.dumps(built["spdx"]) + assert f"LICENSE-gpl3.txt` — {report.NESTED_ROLE}" in built["notices"] + + +def test_a_header_the_depfile_says_was_read_keeps_its_terms(workspace, beta): + synth.write(beta / "include/beta/fmt/format.h", "#pragma once\n") + synth.write(beta / "include/beta/fmt/LICENSE", BUNDLED) + synth.write(beta / "include/beta/unused/LICENSE", BUNDLED) + synth.write(beta / "docs/LICENSE", BUNDLED) + _depfiles(workspace, read=(beta / "include/beta/fmt/format.h",)) + + kept = _kept(workspace) + + assert kept["beta-src/include/beta/fmt/LICENSE"] == { + "how": "header-read", + "path": "beta-src/include/beta/fmt/format.h", + "member": EXTENSION, + } + assert kept["beta-src/include/beta/unused/LICENSE"] is None + assert kept["beta-src/docs/LICENSE"] is None + + +def test_a_depfile_overrules_the_include_directory(workspace, beta): + synth.write(beta / "include/COPYING", BUNDLED) + _depfiles(workspace) + + assert _kept(workspace)["beta-src/include/COPYING"] is None + + +def _no_record(workspace) -> None: + """No dependency output at all.""" + + +def _one_record_short(workspace) -> None: + """module.cpp includes alpha.cpp, which has no object record of its own.""" + module, alpha = _compiled(workspace) + synth.depfile(workspace.build, "ext", module, (alpha,)) + + +def _placeholder(workspace) -> None: + """The compiler_depend.make CMake writes before the first build.""" + synth.compiler_depend(workspace.build, "ext", {}) + + +@pytest.mark.parametrize("records", [_no_record, _one_record_short, _placeholder]) +def test_without_a_record_per_object_include_directories_stand_in( + workspace, beta, records +): + synth.write(beta / "include/COPYING", BUNDLED) + synth.write(beta / "include/beta/fmt/LICENSE", BUNDLED) + synth.write(beta / "docs/LICENSE", BUNDLED) + records(workspace) + + kept = _kept(workspace) + + assert kept["beta-src/include/COPYING"]["how"] == "include-dir" + assert kept["beta-src/include/COPYING"]["path"] == "beta-src/include" + # Beneath the include directory too: any header under it may have been read. + assert kept["beta-src/include/beta/fmt/LICENSE"]["how"] == "include-dir" + assert kept["beta-src/docs/LICENSE"] is None + + +def test_a_stale_depfile_is_ignored_and_a_precompiled_header_counts(workspace, beta): + for name in ("stale", "pch", "unity"): + synth.write(beta / f"{name}/{name}.h", "#pragma once\n") + synth.write(beta / f"{name}/LICENSE", BUNDLED) + _depfiles(workspace) + # A source the target no longer compiles, and a header CMake generated into + # the objects directory to precompile. + synth.depfile( + workspace.build, + "ext", + (workspace.root / "src/cpp/removed.cpp").resolve(), + ((beta / "stale/stale.h").resolve(),), + ) + synth.depfile( + workspace.build, + "ext", + (workspace.build / "CMakeFiles/ext.dir/cmake_pch.hxx").resolve(), + ((beta / "pch/pch.h").resolve(),), + ) + # A unity source left from a build the codemodel no longer describes. + synth.depfile( + workspace.build, + "ext", + (workspace.build / "CMakeFiles/ext.dir/Unity/unity_0_cxx.cxx").resolve(), + ((beta / "unity/unity.h").resolve(),), + ) + + kept = _kept(workspace) + + assert kept["beta-src/stale/LICENSE"] is None + assert kept["beta-src/unity/LICENSE"] is None + assert kept["beta-src/pch/LICENSE"]["how"] == "header-read" + + +def test_an_include_directory_holding_the_whole_tree_keeps_its_terms(workspace, beta): + synth.write(beta / "vendor/LICENSE", BUNDLED) + # `-I _deps` credits every fetched tree, so each needs its own terms. + synth.write(_deps(workspace) / "license-list-data-src/LICENSE", BUNDLED) + synth.add_include(workspace.build, "ext", workspace.build / "_deps") + + assert _kept(workspace)["beta-src/vendor/LICENSE"] == { + "how": "include-dir", + "path": "beta-src", + "member": EXTENSION, + } + + +def test_without_dependency_output_listed_sources_stand_in(workspace, beta): + synth.write(beta / "src/impl.h", "#pragma once\n") + synth.write(beta / "src/LICENSE", BUNDLED) + synth.add_source(workspace.build, "ext", beta / "src/impl.h") + + assert _kept(workspace)["beta-src/src/LICENSE"]["how"] == "listed-source" + + +def _depfiles_in_sub(workspace, monkeypatch) -> None: + """Depfiles name paths from where make ran the compiler.""" + module, alpha = _compiled(workspace) + header = "../_deps/beta-src/include/beta/fmt/format.h" + synth.depfile(workspace.build, "ext", module, (header,), directory="sub") + synth.depfile(workspace.build, "ext", alpha, directory="sub") + + +def _compiler_depend_in_sub(workspace, monkeypatch) -> None: + """compiler_depend.make names paths inside the build from its top.""" + module, _ = _compiled(workspace) + objects = "sub/CMakeFiles/ext.dir" + header = "_deps/beta-src/include/beta/fmt/format.h" + synth.compiler_depend( + workspace.build, + "ext", + { + f"{objects}/module.cpp.o": (str(module), header), + f"{objects}/alpha.cpp.o": ("_deps/alpha-src/alpha.cpp",), + }, + directory="sub", + ) + + +def _ninja_log_in_sub(workspace, monkeypatch) -> None: + """Ninja's deps log names paths from the top of the build.""" + module, _ = _compiled(workspace) + objects = "sub/CMakeFiles/ext.dir" + _ninja( + workspace, + monkeypatch, + f"{objects}/module.cpp.o: #deps 2, deps mtime 1 (VALID)\n" + f" {module}\n _deps/beta-src/include/beta/fmt/format.h\n\n" + f"{objects}/alpha.cpp.o: #deps 1, deps mtime 1 (VALID)\n" + " _deps/alpha-src/alpha.cpp\n\n", + ) + + +@pytest.mark.parametrize( + "records", [_depfiles_in_sub, _compiler_depend_in_sub, _ninja_log_in_sub] +) +def test_relative_paths_in_dependency_output_resolve_as_written( + workspace, beta, records, monkeypatch +): + synth.write(beta / "include/beta/fmt/LICENSE", BUNDLED) + synth.set_directory(workspace.build, "ext", "sub") + records(workspace, monkeypatch) + + assert _kept(workspace)["beta-src/include/beta/fmt/LICENSE"]["how"] == "header-read" + + +def test_a_linked_archive_keeps_every_nested_text_of_its_component(workspace, beta): + synth.write(beta / "lib/libbeta.a", synth.ar_archive({"b.o": synth.elf_object()})) + synth.write(beta / "vendor/LICENSE", BUNDLED) + synth.link_line(workspace.build, "ext", (str(beta / "lib/libbeta.a"),)) + _depfiles(workspace) + + assert _kept(workspace)["beta-src/vendor/LICENSE"] == { + "how": "prebuilt-library", + "path": "beta-src", + "member": EXTENSION, + } + + +def test_an_archive_owned_outside_the_component_trees_keeps_them_too( + workspace, beta, tmp_path +): + prebuilt = tmp_path / "prebuilt" + synth.write(prebuilt / "libbeta.a", synth.ar_archive({"b.o": synth.elf_object()})) + synth.write(beta / "vendor/LICENSE", BUNDLED) + synth.link_line(workspace.build, "ext", (str(prebuilt / "libbeta.a"),)) + _depfiles(workspace) + space = evidence_module.discover(workspace.root, workspace.build) + space.ownership.register_root(prebuilt, "beta") + + evidence = synth.collect(workspace, discovery=space)["build-evidence"] + + nested = next( + item + for item in evidence["components"]["beta"]["evidence"] + if item["path"] == "beta-src/vendor/LICENSE" + ) + assert nested["reached_by"]["how"] == "prebuilt-library" + + +def test_a_header_read_from_the_build_tree_is_its_source_file(workspace, beta): + built = _deps(workspace) / "beta-build" + synth.write(beta / "include/beta/fmt/format.h", "// fmt\n") + synth.write(beta / "include/beta/fmt/LICENSE", BUNDLED) + synth.write(beta / "gen/LICENSE", BUNDLED) + # A copy staged elsewhere, and a header generated where its tree would hold it. + synth.write(built / "staged/format.h", "// fmt\n") + synth.write(built / "gen/config.h", "#define GENERATED 1\n") + _depfiles(workspace, read=(built / "staged/format.h", built / "gen/config.h")) + + kept = _kept(workspace) + + assert kept["beta-src/include/beta/fmt/LICENSE"]["path"] == ( + "beta-src/include/beta/fmt/format.h" + ) + assert kept["beta-src/gen/LICENSE"]["path"] == "beta-src/gen/config.h" + + +def test_a_component_path_outside_its_trees_is_not_reach(workspace, alpha, tmp_path): + synth.write(alpha / "fmt/LICENSE", BUNDLED) + elsewhere = tmp_path / "other" / "_deps" / "alpha-src" / "fmt" / "format.h" + _depfiles(workspace, read=(elsewhere,)) + + assert _kept(workspace)["alpha-src/fmt/LICENSE"] is None + + +def test_a_build_tree_file_keeps_its_place_and_names_only_its_own_sources( + workspace, alpha, beta, tmp_path +): + deps = _deps(workspace) + synth.write(beta / "include/beta/fmt/format.h", "// fmt\n") + synth.write(alpha / "other/other.h", "// other\n") + synth.write(deps / "beta-build/gen/format.h", "// fmt\n") + synth.write(deps / "beta-build/gen/other.h", "// other\n") + synth.write(beta / "a/dup.h", "// dup\n") + synth.write(beta / "b/dup.h", "// dup\n") + # Empty, like many files of any tree: the bytes say nothing about origin. + synth.write(beta / "empty/blank.h", "") + synth.write(deps / "beta-build/gen/blank.h", "") + # A source tree reached through a link has its `-build` sibling beside the link. + real = tmp_path / "elsewhere" / "linked" + real.mkdir(parents=True) + (deps / "linked-src").symlink_to(real) + synth.write(deps / "linked-build/gen/config.h", "#define X 1\n") + space = evidence_module.discover(workspace.root, workspace.build) + resolver = inventory_module.Resolver(space) + + def shown(key, path): + return sorted(resolver._displayed({key: {path.resolve(): "header-read"}})[key]) + + assert shown("beta", deps / "beta-build/gen/format.h") == [ + "beta-src/gen/format.h", + "beta-src/include/beta/fmt/format.h", + ] + assert shown("beta", deps / "beta-build/gen/other.h") == ["beta-src/gen/other.h"] + assert shown("beta", deps / "beta-build/gen/blank.h") == ["beta-src/gen/blank.h"] + # A source-tree file is itself, whatever else holds its bytes. + assert shown("beta", beta / "a/dup.h") == ["beta-src/a/dup.h"] + assert shown("linked", deps / "linked-build/gen/config.h") == [ + "linked-src/gen/config.h" + ] + + +def test_reach_is_attributed_with_the_ownership_asked_about(workspace): + config = _deps(workspace) / "beta-build/gen/config.h" + synth.write(config, "#define X 1\n") + _depfiles(workspace, read=(config,)) + space = evidence_module.discover(workspace.root, workspace.build) + artifact = space.graph.by_name(EXTENSION.rsplit("/", 1)[-1])[0] + + # Only the ownership index knows a `-build` tree's owner. + assert "beta" not in space.graph.reached(artifact) + assert "beta" in space.graph.reached(artifact, space.ownership) + + +def test_a_build_tool_or_custom_target_is_no_reach(workspace, beta): + synth.write(beta / "codegen/LICENSE", BUNDLED) + synth.write(beta / "scripts/LICENSE", BUNDLED) + synth.add_target( + workspace.build, + "beta_codegen", + "EXECUTABLE", + (str(beta / "codegen/main.cpp"),), + "ext", + ) + synth.add_target( + workspace.build, + "beta_generate", + "UTILITY", + (str(beta / "scripts/generate.py"),), + "ext", + ) + _depfiles(workspace) + + kept = _kept(workspace) + + assert kept["beta-src/codegen/LICENSE"] is None + assert kept["beta-src/scripts/LICENSE"] is None + + +def test_a_reuse_sidecar_covers_only_its_file(workspace, alpha): + synth.write(alpha / "third/bundled.cpp", "// bundled\n") + synth.write(alpha / "third/data.txt", "data\n") + synth.write(alpha / "third/data.txt.license", "SPDX-FileCopyrightText: 2020 A\n") + synth.write(alpha / "third/used.h", "#pragma once\n") + synth.write(alpha / "third/used.h.license", "SPDX-FileCopyrightText: 2020 B\n") + synth.add_source(workspace.build, "ext", alpha / "third/bundled.cpp") + _depfiles( + workspace, read=(alpha / "third/used.h",), extra=(alpha / "third/bundled.cpp",) + ) + + kept = _kept(workspace) + + assert kept["alpha-src/third/data.txt.license"] is None + assert kept["alpha-src/third/used.h.license"] == { + "how": "header-read", + "path": "alpha-src/third/used.h", + "member": EXTENSION, + } + + +def test_copied_and_derived_members_keep_the_terms_beside_them(workspace): + vendorpy = _deps(workspace) / "vendorpy-src" + synth.write(vendorpy / "pkg/LICENSE", BUNDLED) + synth.write(vendorpy / "pkg/assets/NOTICE", BUNDLED) + synth.write(vendorpy / "docs/LICENSE", BUNDLED) + + kept = _kept(workspace) + + assert kept["vendorpy-src/pkg/LICENSE"] == { + "how": "copied-file", + "path": "vendorpy-src/pkg/mod.py", + "member": "vendorpy/mod.py", + } + assert kept["vendorpy-src/pkg/assets/NOTICE"]["how"] == "derived-file" + assert kept["vendorpy-src/docs/LICENSE"] is None + + +def _ninja(workspace, monkeypatch, log: str) -> None: + synth.set_generator(workspace.build, "Ninja") + asked = discovery._ask + monkeypatch.setattr( + discovery, + "_ask", + lambda *command: log if command[0] == "ninja" else asked(*command), + ) + + +def test_ninja_deps_log_is_read_for_what_a_target_read( + workspace, monkeypatch, caplog, beta +): + synth.write(beta / "include/beta/fmt/LICENSE", BUNDLED) + header = (beta / "include/beta/fmt/format.h").resolve() + _ninja( + workspace, + monkeypatch, + "".join( + f"CMakeFiles/ext.dir/{source.name}.o: #deps 2, deps mtime 1 (VALID)\n" + f" {source}\n {header}\n\n" + for source in _compiled(workspace) + ) + # An output whose depfile listed nothing, as a precompiled header's can. + + "CMakeFiles/ext.dir/cmake_pch.hxx.gch: #deps 0, deps mtime 1 (VALID)\n\n", + ) + + with caplog.at_level(logging.WARNING, logger="isaaccapture.sbom"): + kept = _kept(workspace) + + assert kept["beta-src/include/beta/fmt/LICENSE"]["how"] == "header-read" + assert not [item for item in caplog.records if "ninja -t deps" in item.message] + + +def test_ninja_records_count_only_for_their_own_objects_directory( + workspace, monkeypatch, beta +): + synth.write(beta / "other/LICENSE", BUNDLED) + other = (beta / "other/other.h").resolve() + module, _ = _compiled(workspace) + _ninja( + workspace, + monkeypatch, + "".join( + f"CMakeFiles/ext.dir/{source.name}.o: #deps 1, deps mtime 1 (VALID)\n" + f" {source}\n\n" + for source in _compiled(workspace) + ) + # Another target compiling the same source with other headers. + + "sub/CMakeFiles/other.dir/module.cpp.o: #deps 2, deps mtime 1 (VALID)\n" + f" {module}\n {other}\n\n", + ) + + assert _kept(workspace)["beta-src/other/LICENSE"] is None + + +def test_without_ninja_the_fallback_is_announced_once( + workspace, monkeypatch, caplog, beta +): + synth.write(beta / "include/COPYING", BUNDLED) + _ninja(workspace, monkeypatch, "") + + with caplog.at_level(logging.WARNING, logger="isaaccapture.sbom"): + kept = _kept(workspace) + + assert kept["beta-src/include/COPYING"]["how"] == "include-dir" + assert ( + len([item for item in caplog.records if "ninja -t deps" in item.message]) == 1 + ) + + +def test_depfile_rules_are_read_as_make_reads_them(): + text = ( + "# CMAKE generated file: DO NOT EDIT!\n" + "obj.o: /src/a\\ b.cpp \\\n /inc/x$$y.h \\\n /inc/z.h\n" + "\n/inc/z.h:\n" + ) + + assert discovery._make_rules(text) == [["/src/a b.cpp", "/inc/x$y.h", "/inc/z.h"]] + + +def test_registering_an_owner_forgets_what_was_asked_before(tmp_path): + ownership = discovery.Ownership() + source = tmp_path / "tree" / "a.h" + vendored = tmp_path / "vendor" / "b.h" + assert ownership.owner_of(source) is None + ownership.register_root(tmp_path / "tree", "tree") + assert ownership.owner_of(source) == "tree" + + assert ownership.owner_of(vendored) is None + ownership.register_file(vendored, "vendored:b") + assert ownership.owner_of(vendored) == "vendored:b" + + +def test_a_steward_line_is_recognised_however_its_copy_spells_it(): + notice = "Copyright (C) 2007 Free Software Foundation, Inc. " + + assert licensing.notice_is_the_licence_authors(notice, ["GPL-3.0-only"]) + + +def test_the_evidence_names_the_first_member_that_reached_a_text(workspace, beta): + synth.write(beta / "vendor/LICENSE", BUNDLED) + component = evidence_module.discover(workspace.root, workspace.build).components[ + "beta" + ] + read = {"beta": {"beta-src/vendor/v.h": "header-read"}} + inventory = SimpleNamespace( + attributions={ + member: inventory_module.Attribution( + path=member, origin="built", detail="", read=read + ) + for member in ("z/late.so", "a/early.so") + } + ) + + settled = build_module._with_reached_nested(inventory, {"beta": component}) + + kept = next( + item + for item in settled["beta"].evidence + if item.path == "beta-src/vendor/LICENSE" + ) + assert kept.reached_by.member == "a/early.so" diff --git a/tests/python/sbom/test_spdx_conformance.py b/tests/python/sbom/test_spdx_conformance.py index 55518238ba..c03f888acc 100644 --- a/tests/python/sbom/test_spdx_conformance.py +++ b/tests/python/sbom/test_spdx_conformance.py @@ -38,10 +38,7 @@ def test_an_identifier_newer_than_the_grammar_still_validates(workspace): """A vendored file may name an exception the pinned list has and the grammar consumers validate with does not; it travels as a LicenseRef with its texts, and the report lists it as an evidence gap.""" - import json - import synth - from sbom import build as build_module synth.write( workspace.root / "src/vendor/rsync_helper.h", @@ -50,14 +47,12 @@ def test_an_identifier_newer_than_the_grammar_still_validates(workspace): ) synth.track(workspace.root) synth.write_wheel(workspace.wheel, synth.wheel_payload(workspace)) - out_dir = workspace.root / "sbom" - build_module.build(workspace.root, workspace.build, workspace.wheel, out_dir) + collected = synth.collect(workspace) - stem = synth.WHEEL_NAME.removesuffix(".whl") - document = parse_file(str(out_dir / f"{stem}.spdx.json")) + document = parse_file(str(collected["out_dir"] / f"{synth.STEM}.spdx.json")) assert validate_full_spdx_document(document) == [] - report = json.loads((out_dir / f"{stem}.licenses.json").read_text()) + report = collected["licenses"] component = next( item for item in report["components"] diff --git a/tests/python/sbom/test_static_archives.py b/tests/python/sbom/test_static_archives.py index 646646295c..4f3b349bf4 100644 --- a/tests/python/sbom/test_static_archives.py +++ b/tests/python/sbom/test_static_archives.py @@ -5,8 +5,6 @@ from __future__ import annotations -import json - import pytest import synth from sbom import build as build_module @@ -15,7 +13,6 @@ from sbom import evidence as evidence_module EXTENSION = "_ext.cpython-312-x86_64-linux-gnu.so" -STEM = synth.WHEEL_NAME.removesuffix(".whl") ONE_OBJECT = synth.ar_archive({"runtime.o": b"\x7fELF object"}) # An agreement that quotes the notices of what it bundles, as the CUDA EULA does. EULA = ( @@ -24,15 +21,6 @@ ) -def _collect(workspace) -> dict: - out_dir = workspace.root / "sbom" - build_module.build(workspace.root, workspace.build, workspace.wheel, out_dir) - return { - kind: json.loads((out_dir / f"{STEM}.{kind}.json").read_text()) - for kind in ("spdx", "licenses", "build-evidence") - } - - def _no_package(monkeypatch) -> None: """The build machine's package manager owns nothing outside its own trees.""" monkeypatch.setattr( @@ -565,7 +553,7 @@ def test_a_packaged_archive_is_inventoried_through_its_package( ) _link(workspace, str(archive)) - collected = _collect(workspace) + collected = synth.collect(workspace) row = _row(collected, "system:libsquash.a") assert row["license_concluded"] == "Zlib" @@ -617,7 +605,7 @@ def test_a_system_component_names_only_the_holders_its_package_declares( _packaged(monkeypatch, "libsquash-dev", copyright_file) _link(workspace, str(archive)) - collected = _collect(workspace) + collected = synth.collect(workspace) row = _row(collected, "system:libsquash.a") assert _package(row, collected["spdx"])["copyrightText"] == expected @@ -634,7 +622,7 @@ def test_a_cuda_archive_is_a_component_under_the_toolkits_terms( monkeypatch.setattr(discovery, "_toolkit_roots", lambda: [toolkit.resolve()]) _link(workspace, str(archive)) - collected = _collect(workspace) + collected = synth.collect(workspace) key = "system:libcudart_static.a" row = _row(collected, key) @@ -673,7 +661,7 @@ def test_two_archives_sharing_a_name_are_two_components( ) _link(workspace, str(first), str(second)) - collected = _collect(workspace) + collected = synth.collect(workspace) records = collected["build-evidence"]["static_inputs"] keys = {records[str(first)]["component"], records[str(second)]["component"]} @@ -689,7 +677,7 @@ def test_an_archive_with_no_members_is_recorded_not_inventoried( _no_package(monkeypatch) _link(workspace, str(archive)) - collected = _collect(workspace) + collected = synth.collect(workspace) record = collected["build-evidence"]["static_inputs"][str(archive)] assert record["members"] == 0 @@ -721,7 +709,7 @@ def test_a_tracked_archive_is_credited_to_the_holder_reuse_names( ): _tracked_archive(workspace, monkeypatch, "Upstream Archive Co") - collected = _collect(workspace) + collected = synth.collect(workspace) row = _row(collected, "vendored:upstream-archive-co") assert "prebuilt-library" in row["how_it_reaches_the_wheel"] @@ -733,7 +721,7 @@ def test_a_tracked_archive_naming_no_other_holder_is_the_projects( """Every tracked file has REUSE info; one naming nobody else is the project's.""" archive = _tracked_archive(workspace, monkeypatch) - collected = _collect(workspace) + collected = synth.collect(workspace) record = collected["build-evidence"]["static_inputs"][str(archive)] assert record["component"] is None @@ -757,7 +745,7 @@ def test_an_input_nothing_explains_stops_the_build( with pytest.raises( build_module.BuildError, match=r"libmystery\.a \(linked by ext\)" ): - _collect(workspace) + synth.collect(workspace) FMT_LICENCE_HEAD = ( @@ -797,7 +785,7 @@ def test_a_vcpkg_archive_is_credited_to_the_port_that_installed_it( _no_package(monkeypatch) _link(workspace, str(installed / "x64-linux" / "lib" / "libfmt.a")) - collected = _collect(workspace) + collected = synth.collect(workspace) row = _row(collected, "vcpkg:fmt") assert row["kind"] == "vcpkg-port" @@ -872,7 +860,7 @@ def test_a_file_no_port_lists_is_still_unexplained(workspace, monkeypatch): _link(workspace, str(stray)) with pytest.raises(build_module.BuildError, match=r"libstray\.a \(linked by ext\)"): - _collect(workspace) + synth.collect(workspace) def test_an_unlinked_port_is_excluded_as_installed_by_vcpkg(workspace): @@ -885,7 +873,7 @@ def test_an_unlinked_port_is_excluded_as_installed_by_vcpkg(workspace): "madler/zlib", ) - collected = _collect(workspace) + collected = synth.collect(workspace) exclusion = next( item @@ -926,7 +914,7 @@ def test_an_include_directory_one_port_owns_credits_that_port(workspace): def test_a_port_with_several_sources_names_no_upstream(workspace): _eigen_port(workspace.build / "vcpkg_installed") - collected = _collect(workspace) + collected = synth.collect(workspace) row = _row(collected, "vcpkg:eigen3") assert row["version"] == "3.4.0" From 39ef38d260eaff91c652e1d786f8a72660498e9c Mon Sep 17 00:00:00 2001 From: Andrei Aristarkhov Date: Tue, 20 Oct 2026 01:09:52 -0700 Subject: [PATCH 081/101] build(sbom): account for every grant text in a component's conclusion A component's conclusion was built from the ids its grant texts matched, and a LicenseRef stood in only when nothing matched, so a recognised licence hid any text beside it that matched nothing: MIT beside an EULA concluded MIT. Every grant text is now accounted for. Ids a text names are concluded; a text that matches nothing, matches only a tie whose ids share one reference text, needs an id the grammar cannot express, or is explained for less than four fifths of its terms (copyright notices aside) is carried verbatim as the component's LicenseRef beside them. A tie names its plain id where every other id only qualifies it (MPL, OFL, CAL). The verifier now compares licence expressions exactly and holds a shared LicenseRef to the text its id names. Signed-off-by: Andrei Aristarkhov --- docs/source/references/sbom.rst | 24 +- scripts/sbom/sbom/discovery.py | 17 +- scripts/sbom/sbom/document.py | 19 +- scripts/sbom/sbom/licensing.py | 322 ++++++++++++++++++++------- scripts/sbom/sbom/report.py | 50 +++-- scripts/sbom/sbom/validate.py | 29 +-- tests/python/sbom/conftest.py | 20 +- tests/python/sbom/synth.py | 52 +++++ tests/python/sbom/test_gates.py | 27 +-- tests/python/sbom/test_licensing.py | 308 +++++++++++++++++++++++-- tests/python/sbom/test_wheel_sbom.py | 140 ++++++++++-- 11 files changed, 784 insertions(+), 224 deletions(-) diff --git a/docs/source/references/sbom.rst b/docs/source/references/sbom.rst index 58dd6cb979..04386b217b 100644 --- a/docs/source/references/sbom.rst +++ b/docs/source/references/sbom.rst @@ -213,9 +213,10 @@ against, so a dependency arriving under any of them is identified. Naming it in document is a second question: a document states the licence list it was built against, and ``license-expression`` — which a consumer validates with, and which :code-file:`scripts/sbom/pyproject.toml` pins — ships its own, older copy. An id the fetched -list knows and that grammar does not cannot be published as an id, so it travels as a -``LicenseRef-`` carrying the text, and the component is listed under *License evidence gaps* -with that as the stated reason. +list knows and that grammar does not cannot be published as an id, so the text it was read +from travels as a ``LicenseRef-`` — a REUSE declaration naming one travels whole, since an +``OR`` or a ``WITH`` cannot be kept in part — and the component is listed under *License +evidence gaps* with that as the stated reason. What the wheel carries for a third-party component is that component's own license file, verbatim, copyright holders and all. Substituting a reference text would convey the terms @@ -282,10 +283,19 @@ text, not on a document that is mostly one. A file that genuinely carries two *d licenses is reported as carrying both; where one is a near-duplicate of the other, only the match survives, and a disjunctive grant is not expressible here at all. -A text that matches nothing is not guessed at. It becomes a ``LicenseRef-`` whose full text -travels in the document, which says "these are the terms that shipped" without claiming to -know which license they are. Those components are listed in the report under *License -evidence gaps*. +Every grant a component ships is accounted for in its conclusion. A text that matches +nothing is not guessed at: it travels in the document as the component's ``LicenseRef-``, +which says "these are the terms that shipped" without claiming to know which license they +are, AND-ed with what its other texts are identified as — an MIT ``LICENSE`` beside an +unmatched EULA concludes ``LicenseRef- AND MIT``. A text matching identifiers +that share one reference text names the one the others extend by a suffix — ``MPL-2.0`` +beside ``MPL-2.0-no-copyleft-exception``, ``OFL-1.1`` beside its ``-RFN`` variants — since +that is SPDX's name for the unmodified text. Where there is none, as for ``GPL-3.0-only`` +and ``GPL-3.0-or-later``, the choice is the licensor's, stated in a notice and not in the +text, so the text is carried too. So is a text its identifiers account for less than four +fifths of, its copyright notices aside, beside the identifiers it names: the rest may be +terms no identifier names, as in fmt's MIT with its exception paragraph. The ``LicenseRef-`` carries exactly the texts +carried. Those components are listed in the report under *License evidence gaps*. A component that states a licence nowhere at all blocks publication, because packaging the text is the obligation the wheel has to meet. For material checked in here that is any one diff --git a/scripts/sbom/sbom/discovery.py b/scripts/sbom/sbom/discovery.py index 92d08be8e1..2efeb0d916 100644 --- a/scripts/sbom/sbom/discovery.py +++ b/scripts/sbom/sbom/discovery.py @@ -1622,21 +1622,8 @@ def discover_vendored( components: dict[str, Component] = {} for key, paths in grouped.items(): - evidence = licensing.canonical_evidence(key, licences.get(key, set())) - concluded, declared = licensing.expression(evidence, key) - if licences.get(key): - # The declared expressions, structure and all. Rebuilding this - # from the reference texts the identifiers name turns `A OR B` into - # `A AND B` -- a choice restated as both obligations at once. - declared = licensing.combine(sorted(licences[key])) - # An id the pinned list knows and the grammar does not travels as a - # LicenseRef carrying the texts, as `licensing.expression` does. - if declared != "NOASSERTION" and not licensing.grammar_names( - licensing.identifiers_in(declared) - ): - evidence += licensing.exception_evidence(key, declared) - declared = licensing.license_ref(key) - concluded = declared + concluded, evidence = licensing.declared_terms(key, licences.get(key, set())) + declared = concluded # What the build read first, so a shortened list names what ships; say # when it is shortened, or it reads as the whole set. named = [ diff --git a/scripts/sbom/sbom/document.py b/scripts/sbom/sbom/document.py index af60c30fdd..4ece7e95e3 100644 --- a/scripts/sbom/sbom/document.py +++ b/scripts/sbom/sbom/document.py @@ -71,7 +71,7 @@ def _attribution_texts(component: Component, dist_info: str) -> list[str]: + (f" — {report.NESTED_ROLE}" if item.kind == "nested" else "") for item in component.evidence ] - if not any(item.kind == "grant" for item in component.evidence): + if not licensing.grant_texts(component.evidence): texts.append( "No license grant was found in this component's own files; see the " "component-to-license report." @@ -117,14 +117,14 @@ def _extracted_entries(components: list[Component]) -> list[dict]: """One entry per distinct text, named by the id the components now share.""" entries: dict[str, dict] = {} for component in components: - refs = licensing.license_refs( - component.license_concluded, component.license_declared - ) - grants = licensing.obligation_texts(component.evidence) - if not refs or not grants: + carried = licensing.carried_texts(component.evidence) + if not carried: continue body = licensing.verbatim_terms(component.evidence) - for ref in refs: + reason = licensing.why_unnamed(component.evidence) + for ref in licensing.license_refs( + component.license_concluded, component.license_declared + ): if ref in entries and entries[ref]["extractedText"] != body: raise ValueError( f"{ref} was minted for two components with different terms; " @@ -135,7 +135,7 @@ def _extracted_entries(components: list[Component]) -> list[dict]: covers = set(entries.get(ref, {}).get("_for", ())) | {component.name} cited = sorted( set(entries.get(ref, {}).get("_from", ())) - | {item.path for item in grants} + | {item.path for item in carried} ) entries[ref] = { "licenseId": ref, @@ -144,7 +144,8 @@ def _extracted_entries(components: list[Component]) -> list[dict]: "_from": cited, "extractedText": body, "comment": ( - licensing.why_unnamed(grants) + reason[:1].upper() + + reason[1:] + "; reproduced here verbatim from " + ", ".join(cited) + "." diff --git a/scripts/sbom/sbom/licensing.py b/scripts/sbom/sbom/licensing.py index 1d14803387..02499f1106 100644 --- a/scripts/sbom/sbom/licensing.py +++ b/scripts/sbom/sbom/licensing.py @@ -12,6 +12,7 @@ import hashlib import json +import math import re from functools import lru_cache from dataclasses import dataclass, replace @@ -51,12 +52,11 @@ # genuine match sits well above it -- a Debian copyright file naming several # licenses around the one matched is the low end, at a quarter. _COVERAGE_THRESHOLD = 0.20 -# ...and the matches together have to account for most of the file before the -# set of them is called an identification. A Debian copyright file aggregates -# many stanzas, and generic BSD boilerplate clears containment against variants -# the file never mentions; two such matches covering half the text name the file -# no better than its own words do. -_EXPLAINED_THRESHOLD = 0.75 +# A grant whose matches account for less of it than this, its copyright notices +# aside, is carried as well as named: the rest may be terms no id names -- fmt's +# MIT carries an exception paragraph. A holder list is not terms, so it does not +# count against the match. Not a gate on naming. +_EXPLAINED_THRESHOLD = 0.80 # Two licenses are one family when either text is nearly inside the other. _FAMILY_THRESHOLD = 0.90 # REUSE-IgnoreStart @@ -125,8 +125,8 @@ def as_json(self) -> dict: # What counts as a text that discharges the packaging obligation. A REUSE pool # names no single expression for the component and is still a real text, so it -# counts. The build gate, the document, the report and the verifier all ask this -# question; they ask it here. +# counts. The build gate, the report and the verifier all ask this question; +# they ask it here. OBLIGATION_KINDS = frozenset({"grant", "pool"}) @@ -139,15 +139,31 @@ def obligation_texts(evidence) -> list[LicenseEvidence]: return [item for item in evidence if satisfies_obligation(item.kind)] +def grant_texts(evidence) -> list[LicenseEvidence]: + """The texts a component's own licence is read from.""" + return [item for item in evidence if item.kind == "grant"] + + +def carried_texts(evidence) -> list[LicenseEvidence]: + """The grants a component's LicenseRef stands for.""" + return [item.text for item in _reading(evidence).carried] + + +def carried_and_named(evidence) -> list[LicenseEvidence]: + """Carried grants whose ids the conclusion names as well.""" + return _reading(evidence).also_named + + def verbatim_terms(evidence) -> str: - """The body a LicenseRef carries for a component: every text it read. + """The body a LicenseRef carries for a component, from `carried_texts`. One component can state its terms across several files, so this is a join and not a file. The build records its digest and the verifier compares against that, which only works while both sides join the same way -- so they - both call this. + both call this. A component with nothing carried states every grant. """ - return "\n\n".join(item.text for item in obligation_texts(evidence)) + texts = carried_texts(evidence) or grant_texts(evidence) + return "\n\n".join(item.text for item in texts) _DIGITS = re.compile(r"\d+") @@ -398,8 +414,8 @@ def alternatives(matches) -> list[tuple[str, ...]]: """Matched ids, grouped where one text cannot tell them apart. Family members that tie survive `_dominated` together: GPL-3.0-only and - GPL-3.0-or-later share one reference text, and which applies is a notice's - choice, not the text's. + GPL-3.0-or-later share one reference text. `_name_for` says which id, if + any, such a group names. """ entries = corpus()[1] groups: list[list[str]] = [] @@ -419,6 +435,27 @@ def alternatives(matches) -> list[tuple[str, ...]]: return [tuple(group) for group in groups] +def _name_for(group: tuple[str, ...]) -> str | None: + """The id a text matching this group of `alternatives` names, if any. + + Where every other id extends one by a suffix -- MPL-2.0 and its + no-copyleft-exception variant, OFL-1.1 and its -RFN and -no-RFN -- that one + is SPDX's id for the unmodified text, and a notice selects the variant. + -only and -or-later have no such id: naming either asserts a choice. + """ + plain = [ + base + for base in group + if all(other == base or other.startswith(f"{base}-") for other in group) + ] + return plain[0] if len(plain) == 1 else None + + +def _either(names) -> str: + """One text matching references it cannot choose between, for a reader.""" + return "matches the reference text of " + " or ".join(names) + + def matched_prose(matches, name=lambda match: match[0], join=" AND ", lead=""): """What a text matched, for a reader: a tie is one text matching either reference, never both licences at once. `name` renders one match, and @@ -427,13 +464,12 @@ def matched_prose(matches, name=lambda match: match[0], join=" AND ", lead=""): if not matches: return None by_id = {match[0]: match for match in matches} - groups = alternatives(matches) - single = [name(by_id[group[0]]) for group in groups if len(group) == 1] + groups = [(group, _name_for(group)) for group in alternatives(matches)] + single = [name(by_id[plain]) for _, plain in groups if plain] prose = [ - "matches the reference text of " - + " or ".join(name(by_id[item]) for item in group) - for group in groups - if len(group) > 1 + _either(name(by_id[item]) for item in group) + for group, plain in groups + if not plain ] parts = ([lead + join.join(single)] if single else []) + prose return ("; " if prose else join).join(parts) @@ -535,7 +571,7 @@ def discover_in_tree( found.append(item) # "nested" never satisfies the grant test below; that is the point. - if not any(item.kind == "grant" for item in found): + if not grant_texts(found): found.extend(_readme_fallback(component, root, display_prefix)) return found @@ -604,44 +640,67 @@ def grammar_names(identifiers) -> bool: return all(expressible(item) or is_exception(item) for item in identifiers) -def explains_the_text(matches) -> bool: - """Whether the matched licences account for most of what the file says. - - Not a test of whether an identification is right -- a dual-licence file - matches correctly and scores low -- only of whether it explains the file, - which is what the document has to say when it cannot name one. - """ - return sum(coverage for _, _, coverage in matches) >= _EXPLAINED_THRESHOLD +def _explained(text: str, groups) -> float: + """How much of a text its matched `alternatives` groups account for, each + counted once and the text's copyright notices left out.""" + candidate = _ngrams(_normalize(_without_notices(text))) + if not candidate: + return 1.0 + entries = corpus()[1] + return sum( + max(len(entries[key].grams & candidate) for key in group) / len(candidate) + for group in groups + ) -def why_unnamed(grants: list[LicenseEvidence]) -> str: - """Why terms travel verbatim rather than under an SPDX identifier.""" - if grants and all(item.origin == SPDX_REFERENCE for item in grants): - # Declared, not matched: the declaration names an id the grammar lacks. - newer = sorted( - item.identified - for item in grants - if item.identified and not grammar_names([item.identified]) - ) - return ( - f"Declares {', '.join(newer) or 'an identifier'}, newer than the SPDX " - "grammar this document ships with" - ) - if not any(item.matches for item in grants): - return "Matched no text in the SPDX reference corpus" - partial = [item for item in grants if not explains_the_text(item.matches)] - if partial: - share = max( - sum(coverage for _, _, coverage in item.matches) for item in partial - ) - return ( - "What matched in the SPDX reference corpus accounts for only " - f"{share:.0%} of this text, so the matches name part of it and not " - "the whole" +def _without_notices(text: str) -> str: + """The text without the lines `read_notices` takes its notices from.""" + notices = [_normalize(notice) for notice in read_notices(text)] + return "\n".join( + line + for line in text.splitlines() + if not (words := _normalize(line)) + or not any( + notice[index : index + len(words)] == words + for notice in notices + for index in range(len(notice) - len(words) + 1) ) - # Matched, but under an id the grammar shipped with this document cannot - # validate, so it cannot be named here. - return "Matched an identifier newer than the SPDX grammar this document ships with" + ) + + +_NEWER = "newer than the SPDX grammar this document ships with" + + +def why_unnamed(evidence) -> str: + """Why a component's carried grants travel verbatim, as lower-case clauses.""" + carried = _reading(evidence).carried + reasons = [] + if any(grant.unmatched for grant in carried): + reasons.append("matched no text in the SPDX reference corpus") + reasons += [ + f"{_either(group)}, and the text alone does not say which applies" + for group in sorted({group for grant in carried for group in grant.tied}) + ] + reasons += [ + # Rounded down: "only 75%" beside a 75% threshold reads as a contradiction. + "what matched in the SPDX reference corpus accounts for only " + f"{math.floor(grant.explained * 100)}% of {grant.text.path}, its " + "copyright notices aside" + for grant in carried + if grant.partial + ] + declared = sorted( + {key for grant in carried if grant.declared for key in grant.newer} + ) + matched = sorted( + {key for grant in carried if not grant.declared for key in grant.newer} + - set(declared) + ) + if declared: + reasons.append(f"declares {', '.join(declared)}, {_NEWER}") + if matched: + reasons.append(f"matched {', '.join(matched)}, {_NEWER}") + return "; ".join(reasons) def _listed() -> _Listed: @@ -739,48 +798,133 @@ def why_unlicensed(expressions) -> str | None: return None -def expression( - evidence: list[LicenseEvidence], component_key: str | None = None -) -> tuple[str, str]: +@dataclass(frozen=True) +class _Grant: + """One grant as read: the ids it names, split by whether the grammar can + express them, the tied groups it names none for, and how much of it its + matches account for, a tie counted once.""" + + text: LicenseEvidence + named: tuple[str, ...] = () + newer: tuple[str, ...] = () + tied: tuple[tuple[str, ...], ...] = () + explained: float = 0.0 + + @property + def declared(self) -> bool: + """A reference text for an id REUSE declares, not the component's own.""" + return self.text.origin == SPDX_REFERENCE + + @property + def unmatched(self) -> bool: + return not self.declared and not self.text.matches + + @property + def partial(self) -> bool: + return bool(self.text.matches) and self.explained < _EXPLAINED_THRESHOLD + + +@dataclass(frozen=True) +class _Reading: + named: list[str] + carried: list[_Grant] + # Carried grants whose ids the conclusion names too. + also_named: list[LicenseEvidence] + + +def _read_grant(item: LicenseEvidence) -> _Grant: + if item.origin == SPDX_REFERENCE: + # Named by the identifier it was fetched for; matching it against the + # corpus it came from would be circular. + ids, tied, explained = [item.identified] if item.identified else [], (), 1.0 + else: + groups = alternatives(item.matches) + names = {group: _name_for(group) for group in groups} + ids = [plain for plain in names.values() if plain] + tied = tuple(group for group, plain in names.items() if not plain) + explained = _explained(item.text, groups) + return _Grant( + item, + tuple(key for key in ids if grammar_names([key])), + tuple(key for key in ids if not grammar_names([key])), + tied, + explained, + ) + + +def _reading(evidence) -> _Reading: + """Every grant names the ids it can and is carried verbatim where its ids + do not say everything: it matches nothing, ties without a plain id, is + mostly unmatched, or needs an id the grammar cannot express. A REUSE + declaration keeps its structure, an OR or a WITH, so one such id carries + every text it declares.""" + read = [_read_grant(item) for item in grant_texts(evidence)] + whole = any(grant.declared and grant.newer for grant in read) + carried = [ + grant + for grant in read + if (whole and grant.declared) + or ( + not grant.declared + and (grant.newer or grant.tied or grant.unmatched or grant.partial) + ) + ] + + def names(grant: _Grant) -> tuple[str, ...]: + return () if whole and grant.declared else grant.named + + return _Reading( + sorted({key for grant in read for key in names(grant)}), + carried, + [grant.text for grant in carried if names(grant)], + ) + + +def declared_terms( + component_key: str, expressions +) -> tuple[str, list[LicenseEvidence]]: + """A vendored component's terms as its REUSE info declares them, and the + reference texts that convey them. + + The expressions keep their structure -- rebuilding them from the texts + turns `A OR B` into `A AND B`. Where `_reading` carries a text, the whole + declaration travels as the component's LicenseRef, its exceptions' texts + with it. + """ + declared = combine(sorted(expressions)) + evidence = canonical_evidence(component_key, set(expressions)) + if declared == NOASSERTION: + return declared, evidence + exceptions = exception_evidence(component_key, declared) + if _reading(evidence + exceptions).carried: + return license_ref(component_key), evidence + exceptions + return declared, evidence + + +def expression(evidence: list[LicenseEvidence], component_key: str) -> tuple[str, str]: """Concluded and declared expressions for one component. Concluded names what was matched in a text that ships with the artifact. A - grant that matches nothing is not dropped and not guessed at: it becomes a - LicenseRef whose text travels in the document, which says "these are the - terms we shipped" without claiming to know which license they are. - - An ID the shipped grammar cannot express is treated the same way, and the - whole expression falls back rather than just the offending term: dropping - one ID from `A AND B` would understate the terms. + grant `_reading` carries is not dropped and not guessed at: it adds the + component's LicenseRef, whose text travels in the document, which says + "these are the terms we shipped" without claiming to know which license + they are. """ - grants = [item for item in evidence if item.kind == "grant"] - identified = sorted( - {license_id for item in grants for license_id, _, _ in item.matches} - # A reference text is named by the identifier it was fetched for; there - # is no match to read, because matching it would be circular. - | { - item.identified - for item in grants - if item.origin == SPDX_REFERENCE and item.identified - } + reading = _reading(evidence) + identified = reading.named + ( + [license_ref(component_key)] if reading.carried else [] ) - if identified and component_key and not grammar_names(identified): - identified = [license_ref(component_key)] - if not identified and grants and component_key: - identified = [license_ref(component_key)] - concluded = _combine(identified) if identified else "NOASSERTION" + concluded = _combine(identified) if identified else NOASSERTION # Declared is the same expression, because the text it was read from is the # component's own LICENSE -- shipping that file is how authors declare a - # licence, so one artifact answers both questions. It is not an echo: a - # component that ships no grant declares nothing and gets NOASSERTION. + # licence, so one artifact answers both questions. # # Never from a REUSE tag on a grant file. OpenXR's COPYING.adoc is prose # about which licences the project uses, tagged CC-BY-4.0 for the prose, and # promoting that declared a linked Apache-2.0 SDK under a documentation # licence. - declared = concluded if grants else NOASSERTION - return concluded, declared + return concluded, concluded def _combine(expressions: list[str]) -> str: @@ -1226,12 +1370,20 @@ def shared_refs(components) -> dict[str, str]: for text, keys in by_text.items(): if len(keys) < 2: continue - canonical = f"LicenseRef-shared-text-{sha256_text(text)[:12]}" + canonical = shared_ref(text) for key in keys: renamed[license_ref(key)] = canonical return renamed +SHARED_REF = "LicenseRef-shared-text-" + + +def shared_ref(text: str) -> str: + """The id components carrying byte-identical terms share, named by them.""" + return SHARED_REF + sha256_text(text)[:12] + + def sha256_text(text: str) -> str: return hashlib.sha256(text.encode("utf-8")).hexdigest() diff --git a/scripts/sbom/sbom/report.py b/scripts/sbom/sbom/report.py index 0ed2df77b0..74174b1daa 100644 --- a/scripts/sbom/sbom/report.py +++ b/scripts/sbom/sbom/report.py @@ -190,9 +190,29 @@ def notices_markdown( return "\n".join(lines) -def _lowercase_first(text: str) -> str: - """Run a sentence on; its ids keep their case.""" - return text[:1].lower() + text[1:] +def _carried_subject(carried: int, total: int) -> str: + """How many of a component's grants travel verbatim, as a sentence subject.""" + if carried == total == 1: + return "the shipped grant is" + if carried == total: + return f"its {total} shipped grants are" + return f"{carried} of its {total} shipped grants {'is' if carried == 1 else 'are'}" + + +def _carried_reason(evidence) -> str: + """Which of a component's grants travel verbatim, whether their ids are + named too, and why they travel.""" + carried = licensing.carried_texts(evidence) + named = len(licensing.carried_and_named(evidence)) + subject = _carried_subject(len(carried), len(licensing.grant_texts(evidence))) + manner = ( + " rather than named" + if not named + else " as well as named" + if named == len(carried) + else f", {named} of them as well as named" + ) + return f"{subject} carried verbatim{manner}: {licensing.why_unnamed(evidence)}" def report(inventory: Inventory, evidence_doc: dict, wheel_name: str) -> dict: @@ -202,7 +222,8 @@ def report(inventory: Inventory, evidence_doc: dict, wheel_name: str) -> dict: unidentified = [] for key, component in inventory.components_present.items(): - grants = licensing.obligation_texts(component.evidence) + obligations = licensing.obligation_texts(component.evidence) + carried = licensing.carried_texts(component.evidence) components.append( { "component": key, @@ -225,26 +246,23 @@ def report(inventory: Inventory, evidence_doc: dict, wheel_name: str) -> dict: # Same test as the build gate and the verifier: a REUSE pool is a real # text. Requiring a grant here advertised "no licence evidence" for a # component the gates deliberately accepted on the pool they packaged. - if not grants: + if not obligations: no_evidence.append( { "component": key, "reason": "no license grant found in the component's own files", } ) - elif "LicenseRef-" in component.license_concluded: - # Keyed on the conclusion, not on whether anything matched: a text - # can match and still be unusable, when the id it matched is newer - # than the expression grammar a consumer validates with. Either way - # the terms are only readable as the text this wheel carries. + elif carried: + # Not whether anything matched: a text can match and still be + # carried (`licensing.carried_texts`), and one beside it can match + # and be named. Either way these terms are only readable as the + # text this wheel carries. unidentified.append( { "component": key, - "reason": ( - "the shipped grant is carried verbatim rather than named: " - + _lowercase_first(licensing.why_unnamed(grants)) - ), - "evidence": [item.path for item in grants], + "reason": _carried_reason(component.evidence), + "evidence": [item.path for item in carried], } ) @@ -376,7 +394,7 @@ def report_markdown(payload: dict) -> str: lines += [f"- `{item['component']}` — {item['reason']}" for item in gaps] lines.append("") if unidentified: - lines += ["License text shipped but not identified:", ""] + lines += ["License text carried verbatim:", ""] lines += [ f"- `{item['component']}` — {item['reason']}" for item in unidentified ] diff --git a/scripts/sbom/sbom/validate.py b/scripts/sbom/sbom/validate.py index fc374aec5d..1dac3b8c70 100644 --- a/scripts/sbom/sbom/validate.py +++ b/scripts/sbom/sbom/validate.py @@ -64,6 +64,7 @@ def _record_for(manifest: dict, filename: str) -> dict: return record +_SHARED_ID = re.compile(re.escape(licensing.SHARED_REF) + "[0-9a-f]{12}") _EVIDENCE_DIGEST = re.compile( r"\(sha256:(?P[0-9a-f]{64})[^)]*\) packaged at (?P\S+)" ) @@ -408,13 +409,12 @@ def _document_agrees_with_evidence( ): expected = component.get(stated) actual = package.get(field) + # Exactly, LicenseRef ids included: the build settles shared ids + # before it renders either side, and the text check below ties a + # shared id to the text it names and a minted one to its owner, + # never either to the package naming it. if not expected or actual == expected: continue - # Both unnameable is agreement: components sharing one text share one - # LicenseRef, so the document's id is not the evidence's. The text - # behind it is checked below, which is what the id stands for. - if "LicenseRef-" in str(actual) and "LicenseRef-" in expected: - continue failures.append( f"{key}: the document says {field} {actual!r}, " f"the build recorded {expected!r}" @@ -432,18 +432,21 @@ def _document_agrees_with_evidence( } any_stated = {digest for digest in stated.values() if digest} for extracted in spdx.get("hasExtractedLicensingInfos", []): + ref, text = extracted["licenseId"], extracted.get("extractedText", "") + # A shared id names its text, so it is held to that text, not just to + # the terms some component has. + if _SHARED_ID.fullmatch(ref) and ref != licensing.shared_ref(text): + failures.append(f"{ref}: the text it carries is not the one its id names") + continue if not any_stated: # Evidence written before this digest existed cannot answer. - break - digest = hashlib.sha256( - extracted.get("extractedText", "").encode("utf-8") - ).hexdigest() - owner = by_ref.get(extracted["licenseId"]) + continue + owner = by_ref.get(ref) expected = {stated[owner]} if owner and stated.get(owner) else any_stated - if digest not in expected: + if hashlib.sha256(text.encode("utf-8")).hexdigest() not in expected: failures.append( - f"{extracted['licenseId']}: the text it carries is not the terms " - "the build read" + (f" for {owner}" if owner else "") + f"{ref}: the text it carries is not the terms the build read" + + (f" for {owner}" if owner else "") ) return failures diff --git a/tests/python/sbom/conftest.py b/tests/python/sbom/conftest.py index 8cc7bc4eb4..04c4ddaef4 100644 --- a/tests/python/sbom/conftest.py +++ b/tests/python/sbom/conftest.py @@ -5,7 +5,6 @@ from __future__ import annotations -import json import os from pathlib import Path @@ -101,21 +100,4 @@ def workspace(tmp_path): @pytest.fixture def built(workspace): """The workspace after the collector has packaged evidence into the wheel.""" - collected = synth.collect(workspace) - out_dir = collected["out_dir"] - manifest_path = out_dir / "manifest.json" - manifest_path.write_text( - json.dumps(collected["manifest"], indent=2, sort_keys=True) + "\n", - encoding="utf-8", - ) - return { - "workspace": workspace, - "root": workspace.root, - "wheel": workspace.wheel, - "out_dir": out_dir, - "manifest": collected["manifest"], - "manifest_path": manifest_path, - "evidence_path": out_dir / f"{synth.STEM}.build-evidence.json", - "spdx_path": out_dir / f"{synth.STEM}.spdx.json", - "report_path": out_dir / f"{synth.STEM}.licenses.json", - } + return synth.collect_and_bind(workspace) diff --git a/tests/python/sbom/synth.py b/tests/python/sbom/synth.py index 6a7d0851e4..9a6aec272f 100644 --- a/tests/python/sbom/synth.py +++ b/tests/python/sbom/synth.py @@ -1005,6 +1005,58 @@ def collect(workspace: Workspace, **options) -> dict: } +def collect_and_bind(workspace: Workspace, **options) -> dict: + """`collect`, with the manifest written beside it and the paths tests read.""" + collected = collect(workspace, **options) + out_dir = collected["out_dir"] + manifest_path = out_dir / "manifest.json" + manifest_path.write_text( + json.dumps(collected["manifest"], indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) + return { + **collected, + "workspace": workspace, + "root": workspace.root, + "wheel": workspace.wheel, + "manifest_path": manifest_path, + "evidence_path": out_dir / f"{STEM}.build-evidence.json", + "spdx_path": out_dir / f"{STEM}.spdx.json", + "report_path": out_dir / f"{STEM}.licenses.json", + } + + +def members(wheel: Path) -> dict[str, bytes]: + """A wheel's members, RECORD left out so `write_wheel` can restate it.""" + with zipfile.ZipFile(wheel) as archive: + return { + name: archive.read(name) + for name in archive.namelist() + if not name.endswith(".dist-info/RECORD") + } + + +def drop_member(wheel: Path, name: str) -> None: + """Remove one member after the fact, leaving RECORD as it was.""" + with zipfile.ZipFile(wheel) as archive: + payload = { + item: archive.read(item) for item in archive.namelist() if item != name + } + with zipfile.ZipFile(wheel, "w", zipfile.ZIP_DEFLATED) as archive: + for item in sorted(payload): + archive.writestr(item, payload[item]) + + +def replace_member(wheel: Path, name: str, data: bytes) -> None: + """Change one member after the fact, leaving RECORD as it was.""" + with zipfile.ZipFile(wheel) as archive: + payload = {item: archive.read(item) for item in archive.namelist()} + payload[name] = data + with zipfile.ZipFile(wheel, "w", zipfile.ZIP_DEFLATED) as archive: + for item in sorted(payload): + archive.writestr(item, payload[item]) + + def _record_hash(data: bytes) -> str: digest = hashlib.sha256(data).digest() return "sha256=" + base64.urlsafe_b64encode(digest).rstrip(b"=").decode("ascii") diff --git a/tests/python/sbom/test_gates.py b/tests/python/sbom/test_gates.py index f1b6578d48..1f10eff4ba 100644 --- a/tests/python/sbom/test_gates.py +++ b/tests/python/sbom/test_gates.py @@ -148,27 +148,8 @@ def test_a_vendored_library_is_attributed_to_the_package_that_supplied_it( ) -def _replace_member(wheel, name, data): - with zipfile.ZipFile(wheel) as archive: - payload = {item: archive.read(item) for item in archive.namelist()} - payload[name] = data - with zipfile.ZipFile(wheel, "w", zipfile.ZIP_DEFLATED) as archive: - for item in sorted(payload): - archive.writestr(item, payload[item]) - - -def _drop_member(wheel, name): - with zipfile.ZipFile(wheel) as archive: - payload = { - item: archive.read(item) for item in archive.namelist() if item != name - } - with zipfile.ZipFile(wheel, "w", zipfile.ZIP_DEFLATED) as archive: - for item in sorted(payload): - archive.writestr(item, payload[item]) - - def test_editing_a_file_after_the_fact_fails_verification(built): - _replace_member(built["wheel"], "isaaccapture/__init__.py", b"# tampered\n") + synth.replace_member(built["wheel"], "isaaccapture/__init__.py", b"# tampered\n") failures = validate_module.check(built["wheel"]) @@ -177,7 +158,7 @@ def test_editing_a_file_after_the_fact_fails_verification(built): def test_adding_a_file_after_the_fact_fails_verification(built): - _replace_member(built["wheel"], "isaaccapture/surprise.py", b"# added later\n") + synth.replace_member(built["wheel"], "isaaccapture/surprise.py", b"# added later\n") failures = validate_module.check(built["wheel"]) @@ -185,7 +166,7 @@ def test_adding_a_file_after_the_fact_fails_verification(built): def test_mutating_the_wheel_breaks_its_manifest_binding(built): - _replace_member(built["wheel"], "isaaccapture/__init__.py", b"# tampered\n") + synth.replace_member(built["wheel"], "isaaccapture/__init__.py", b"# tampered\n") failures = validate_module.check( built["wheel"], @@ -215,7 +196,7 @@ def test_another_tools_document_beside_ours_is_kept(built): def test_our_own_document_going_missing_is_rejected(built): - _drop_member(built["wheel"], f"{DIST_INFO}/sboms/{WHEEL_NAME}.spdx.json") + synth.drop_member(built["wheel"], f"{DIST_INFO}/sboms/{WHEEL_NAME}.spdx.json") failures = validate_module.check(built["wheel"]) diff --git a/tests/python/sbom/test_licensing.py b/tests/python/sbom/test_licensing.py index 0e256f641c..bfdf7fbdd5 100644 --- a/tests/python/sbom/test_licensing.py +++ b/tests/python/sbom/test_licensing.py @@ -5,9 +5,11 @@ from __future__ import annotations +import math + import pytest import synth -from sbom import discovery, document +from sbom import discovery, document, report from sbom import licensing @@ -120,18 +122,270 @@ def test_a_real_license_survives_surrounding_packaging_boilerplate(): assert [license_id for license_id, _, _ in licensing.identify(wrapped)] == ["Zlib"] +def _grant(key: str, name: str, text: str) -> licensing.LicenseEvidence: + return licensing.evidence_for(key, "component-file", f"{key}/{name}", text, "grant") + + def test_unmatched_grant_becomes_a_license_ref_not_a_guess(): - evidence = [ - licensing.evidence_for( - "demo", "component-file", "demo/LICENSE", synth.PROPRIETARY_LICENSE, "grant" - ) - ] + evidence = [_grant("demo", "LICENSE", synth.PROPRIETARY_LICENSE)] concluded, _ = licensing.expression(evidence, "demo") assert concluded == "LicenseRef-demo" +def test_a_matched_grant_does_not_hide_an_unmatched_one(): + mit = _grant("demo", "LICENSE", synth.spdx_text("MIT")) + eula = _grant("demo", "LICENSE-EULA.txt", synth.PROPRIETARY_LICENSE) + + assert licensing.expression([mit, eula], "demo") == ( + "LicenseRef-demo AND MIT", + "LicenseRef-demo AND MIT", + ) + assert licensing.carried_texts([mit, eula]) == [eula] + assert licensing.verbatim_terms([mit, eula]) == synth.PROPRIETARY_LICENSE + assert licensing.why_unnamed([mit, eula]) == ( + "matched no text in the SPDX reference corpus" + ) + + +_GPL_TIE = ( + "matches the reference text of GPL-3.0-only or GPL-3.0-or-later, and the " + "text alone does not say which applies" +) +_NEWER = "newer than the SPDX grammar this document ships with" + + +def test_a_text_two_ids_share_is_carried_rather_than_named_as_both(): + gpl = _grant("gpl", "COPYING", synth.spdx_text("GPL-3.0-only")) + mit = _grant("gpl", "LICENSE-MIT", synth.spdx_text("MIT")) + eula = _grant("gpl", "EULA", synth.PROPRIETARY_LICENSE) + + assert licensing.expression([gpl], "gpl")[0] == "LicenseRef-gpl" + assert licensing.expression([gpl, mit], "gpl")[0] == "LicenseRef-gpl AND MIT" + assert licensing.carried_texts([gpl, mit]) == [gpl] + assert licensing.why_unnamed([gpl, mit]) == _GPL_TIE + assert licensing.why_unnamed([eula, gpl]) == ( + f"matched no text in the SPDX reference corpus; {_GPL_TIE}" + ) + + +@pytest.mark.parametrize("license_id", ["MPL-2.0", "OFL-1.1"]) +def test_a_tie_with_a_plain_id_names_the_plain_id(license_id): + text = _grant("font", "LICENSE", synth.spdx_text(license_id)) + assert len(licensing.alternatives(text.matches)[0]) > 1, "not a tie any more" + + assert licensing.expression([text], "font")[0] == license_id + assert text.identified == license_id + + +def test_a_tie_the_plain_id_names_is_not_given_as_a_reason(): + both = _grant( + "c", + "COPYING", + synth.spdx_text("MPL-2.0") + "\n\n" + synth.spdx_text("LGPL-2.1-only"), + ) + + assert licensing.expression([both], "c")[0] == "LicenseRef-c AND MPL-2.0" + assert licensing.why_unnamed([both]) == ( + "matches the reference text of LGPL-2.1-only or LGPL-2.1-or-later, and " + "the text alone does not say which applies" + ) + + +def test_a_tied_text_says_how_little_of_it_the_reference_text_is(): + padded = _grant( + "c", "COPYING", synth.spdx_text("LGPL-2.1-only") + _varied_prose(3000) + ) + share = licensing._read_grant(padded).explained + # Counted once it explains too little; counted per id it would not. + assert licensing._EXPLAINED_THRESHOLD / 2 < share < licensing._EXPLAINED_THRESHOLD + + assert licensing.why_unnamed([padded]) == ( + "matches the reference text of LGPL-2.1-only or LGPL-2.1-or-later, and " + "the text alone does not say which applies; what matched in the SPDX " + f"reference corpus accounts for only {math.floor(share * 100)}% of c/COPYING, " + "its copyright notices aside" + ) + + +def _share(text: licensing.LicenseEvidence) -> str: + return f"{math.floor(licensing._read_grant(text).explained * 100)}%" + + +def test_a_mostly_unmatched_grant_is_named_and_carried(): + """The rest of the text may be terms no id names.""" + whole = _grant("demo", "LICENSE", synth.spdx_text("MIT")) + padded = _grant("demo", "COPYING", synth.spdx_text("MIT") + _varied_prose(300)) + zlib = _grant("demo", "COPYING.zlib", synth.spdx_text("Zlib") + _varied_prose(250)) + + assert licensing.expression([whole], "demo")[0] == "MIT" + assert licensing.expression([padded, whole], "demo")[0] == ( + "LicenseRef-demo AND MIT" + ) + assert licensing.carried_texts([padded, whole]) == [padded] + assert licensing.why_unnamed([padded, zlib]) == ( + "what matched in the SPDX reference corpus accounts for only " + f"{_share(padded)} of demo/COPYING, its copyright notices aside; what matched " + f"in the SPDX reference corpus accounts for only {_share(zlib)} of " + "demo/COPYING.zlib, its copyright notices aside" + ) + + +_FMT_EXCEPTION = ( + "As an exception, if, as a result of your compiling your source code, portions " + "of this Software are embedded into a machine-executable object form of such " + "source code, you may redistribute such embedded portions in such object form " + "without including the above copyright and permission notices." +) + + +@pytest.mark.parametrize( + "shape, share, concluded", + [ + # fmt's own shape: named at 0.75, carried at 0.80. + ("exception", (0.75, 0.80), "LicenseRef-demo AND MIT"), + # Two thirds of it: carried at 0.85, named at 0.80. + ("part-exception", (0.80, 0.85), "MIT"), + # A holder list is not terms, however long. + ("holders", (0.99, 1.01), "MIT"), + ], +) +def test_how_much_of_a_grant_must_match_for_its_ids_to_say_it_all( + shape, share, concluded +): + mit = synth.spdx_text("MIT") + text = { + "exception": f"{mit}\n\n--- Optional exception to the license ---\n\n" + + _FMT_EXCEPTION, + "part-exception": mit + "\n\n" + " ".join(_FMT_EXCEPTION.split()[:34]), + "holders": "\n".join( + f"Copyright (c) 2015 Holder Number{index} " + for index in range(8) + ) + + "\n\n" + + mit, + }[shape] + grant = _grant("demo", "LICENSE", text) + assert share[0] < licensing._read_grant(grant).explained < share[1] + + assert licensing.expression([grant], "demo")[0] == concluded + + +@pytest.fixture +def mit_is_newer(monkeypatch): + """MIT as an id the grammar this document ships with cannot express.""" + expressible = licensing.expressible + monkeypatch.setattr( + licensing, "expressible", lambda key: key != "MIT" and expressible(key) + ) + + +def _reference(key: str, license_id: str) -> licensing.LicenseEvidence: + return licensing.evidence_for( + key, + licensing.SPDX_REFERENCE, + f"{license_id}.txt", + synth.spdx_text(license_id), + "grant", + ) + + +@pytest.mark.parametrize( + "texts, concluded, carried, reason", + [ + ( + ["MIT", "Zlib"], + "LicenseRef-demo AND Zlib", + [0], + f"matched MIT, {_NEWER}", + ), + ( + ["MIT", None], + "LicenseRef-demo", + [0, 1], + f"matched no text in the SPDX reference corpus; matched MIT, {_NEWER}", + ), + ( + ["MIT", "GPL-3.0-only"], + "LicenseRef-demo", + [0, 1], + f"{_GPL_TIE}; matched MIT, {_NEWER}", + ), + # A reference text is declared, not unmatched, and its id is not + # restated as matched. + ( + ["reference", "MIT", "Zlib"], + "LicenseRef-demo AND Zlib", + [0, 1], + f"declares MIT, {_NEWER}", + ), + ], + ids=["beside-named", "beside-unmatched", "beside-tie", "declared"], +) +def test_an_id_the_grammar_cannot_name_carries_only_its_texts( + mit_is_newer, texts, concluded, carried, reason +): + evidence = [ + _reference("demo", "MIT") + if source == "reference" + else _grant("demo", f"LICENSE-{index}", synth.PROPRIETARY_LICENSE) + if source is None + else _grant("demo", f"LICENSE-{index}", synth.spdx_text(source)) + for index, source in enumerate(texts) + ] + + assert licensing.expression(evidence, "demo")[0] == concluded + assert licensing.carried_texts(evidence) == [evidence[i] for i in carried] + assert licensing.why_unnamed(evidence) == reason + + +def test_a_declaration_the_grammar_cannot_name_travels_whole(mit_is_newer): + """An OR cannot be kept in part, so every text it declares is carried.""" + declared, evidence = licensing.declared_terms("vendored:x", {"MIT OR Zlib"}) + assert declared == "LicenseRef-vendored-x" + assert [item.identified for item in licensing.carried_texts(evidence)] == [ + "MIT", + "Zlib", + ] + declared, evidence = licensing.declared_terms("vendored:y", {"Zlib OR BSL-1.0"}) + assert declared == "Zlib OR BSL-1.0" + assert licensing.carried_texts(evidence) == [] + + +@pytest.mark.parametrize( + "carried, total, subject", + [ + (1, 1, "the shipped grant is"), + (2, 2, "its 2 shipped grants are"), + (1, 2, "1 of its 2 shipped grants is"), + (2, 3, "2 of its 3 shipped grants are"), + ], +) +def test_the_report_says_how_many_grants_are_carried(carried, total, subject): + assert report._carried_subject(carried, total) == subject + + +def test_the_report_says_whether_a_carried_grant_is_named_too(mit_is_newer): + padded = _grant("demo", "COPYING", synth.spdx_text("Zlib") + _varied_prose(250)) + eula = _grant("demo", "EULA", synth.PROPRIETARY_LICENSE) + _, evidence = licensing.declared_terms("vendored:x", {"MIT OR Zlib"}) + + assert report._carried_reason([padded]) == ( + "the shipped grant is carried verbatim as well as named: what matched in " + f"the SPDX reference corpus accounts for only {_share(padded)} of " + "demo/COPYING, its copyright notices aside" + ) + assert report._carried_reason([padded, eula]).startswith( + "its 2 shipped grants are carried verbatim, 1 of them as well as named: " + ) + # A declaration carried whole names none of its ids. + assert report._carried_reason(evidence) == ( + "its 2 shipped grants are carried verbatim rather than named: declares " + f"MIT, {_NEWER}" + ) + + def test_no_grant_at_all_is_noassertion(): assert licensing.expression([], "demo") == ("NOASSERTION", "NOASSERTION") @@ -208,7 +462,8 @@ def test_a_component_with_no_grant_declares_nothing(tmp_path): def test_a_tag_on_a_documentation_file_is_not_the_package_declaration(tmp_path): """OpenXR's COPYING.adoc is prose about which licences the project uses, tagged CC-BY-4.0 for the prose. Promoting it declared an Apache-2.0 SDK - under a documentation licence.""" + under a documentation licence. The prose matches nothing, so it is carried + as the component's LicenseRef.""" (tmp_path / "LICENSE").write_text(synth.spdx_text("Apache-2.0"), encoding="utf-8") # REUSE-IgnoreStart (tmp_path / "COPYING").write_text( @@ -221,30 +476,37 @@ def test_a_tag_on_a_documentation_file_is_not_the_package_declaration(tmp_path): licensing.discover_in_tree("thing", tmp_path, "thing-src"), "thing" ) - assert concluded == "Apache-2.0" + assert concluded == "Apache-2.0 AND LicenseRef-thing" assert "CC-BY-4.0" not in declared -def test_a_component_carrying_two_licenses_combines_them(): - """Two identified grants must AND together, not collapse to NOASSERTION. +@pytest.mark.parametrize( + "files, concluded", + [ + ({"LICENSE": ("Apache-2.0", "CC-BY-4.0")}, "Apache-2.0 AND CC-BY-4.0"), + ( + {"LICENSE-MIT": ("MIT",), "LICENSE-APACHE": ("Apache-2.0",)}, + "Apache-2.0 AND MIT", + ), + ], +) +def test_two_identified_licenses_combine(files, concluded): + """Two identified licences must AND together, not collapse to NOASSERTION. The combiner returns a parsed expression object, and asking such an object for its truth value raises rather than answering -- so this path has to be exercised with more than one licence. """ evidence = [ - licensing.evidence_for( - "dual", - "component-file", - "dual/LICENSE", - synth.spdx_text("Apache-2.0") + "\n\n" + synth.spdx_text("CC-BY-4.0"), - "grant", - ) + _grant("dual", name, "\n\n".join(map(synth.spdx_text, ids))) + for name, ids in files.items() ] - concluded, _ = licensing.expression(evidence, "dual") - - assert concluded == "Apache-2.0 AND CC-BY-4.0" + assert licensing.expression(evidence, "dual")[0] == concluded + # Nothing carried, so every grant is the terms the build evidence states. + assert licensing.verbatim_terms(evidence) == "\n\n".join( + item.text for item in evidence + ) def test_a_bare_license_file_covers_its_directory(tmp_path): @@ -266,12 +528,10 @@ def test_a_bare_license_file_covers_its_directory(tmp_path): def test_a_text_carrying_two_licences_lists_both_not_a_choice(): - dual = licensing.evidence_for( + dual = _grant( "dual", - "component-file", - "dual/LICENSE", + "LICENSE", synth.spdx_text("Apache-2.0") + "\n\n" + synth.spdx_text("CC-BY-4.0"), - "grant", ) assert dual.identified == "Apache-2.0 AND CC-BY-4.0" diff --git a/tests/python/sbom/test_wheel_sbom.py b/tests/python/sbom/test_wheel_sbom.py index d4e716f84c..ec23e2850b 100644 --- a/tests/python/sbom/test_wheel_sbom.py +++ b/tests/python/sbom/test_wheel_sbom.py @@ -9,6 +9,7 @@ import re import zipfile +import pytest import synth from sbom import build as build_module from sbom import validate as validate_module @@ -152,6 +153,95 @@ def test_an_unidentified_grant_travels_verbatim_in_the_document(built): assert "DEMO CORP" in extracted[ref]["extractedText"] +@pytest.fixture +def eula_beside_mit(workspace): + """alpha ships its MIT LICENSE and an EULA that matches nothing.""" + eula = synth.PROPRIETARY_LICENSE.replace("Demo SDK", "Alpha runtime") + (workspace.build / "_deps" / "alpha-src" / "LICENSE-EULA.txt").write_text( + eula, encoding="utf-8" + ) + synth.track(workspace.root) + return {**synth.collect_and_bind(workspace), "eula": eula} + + +def test_an_unmatched_grant_beside_a_matched_one_is_carried_and_reported( + eula_beside_mit, +): + built = eula_beside_mit + package = next( + item for item in built["spdx"]["packages"] if item["name"] == "alpha" + ) + assert package["licenseConcluded"] == "LicenseRef-alpha AND MIT" + extracted = { + item["licenseId"]: item for item in built["spdx"]["hasExtractedLicensingInfos"] + } + assert extracted["LicenseRef-alpha"]["extractedText"] == built["eula"] + assert extracted["LicenseRef-alpha"]["comment"] == ( + "Matched no text in the SPDX reference corpus; reproduced here verbatim " + "from alpha-src/LICENSE-EULA.txt." + ) + unidentified = { + item["component"]: item + for item in built["licenses"]["components_with_unidentified_license"] + } + assert unidentified["alpha"] == { + "component": "alpha", + "reason": "1 of its 2 shipped grants is carried verbatim rather than " + "named: matched no text in the SPDX reference corpus", + "evidence": ["alpha-src/LICENSE-EULA.txt"], + } + assert ( + validate_module.check( + built["wheel"], + build_evidence=built["evidence_path"], + manifest=built["manifest_path"], + ) + == [] + ) + + +def _rewrite_alpha(built, expression: str) -> list[str]: + """Rewrite alpha's expressions in the embedded document; what check says.""" + member = f"{DIST_INFO}/sboms/{WHEEL_NAME}.spdx.json" + payload = synth.members(built["wheel"]) + spdx = json.loads(payload[member]) + for package in spdx["packages"]: + if package["name"] == "alpha": + package["licenseConcluded"] = package["licenseDeclared"] = expression + payload[member] = json.dumps(spdx).encode() + synth.write_wheel(built["wheel"], payload) + return validate_module.check(built["wheel"], build_evidence=built["evidence_path"]) + + +def _disagreement(expression: str) -> list[str]: + recorded = "LicenseRef-alpha AND MIT" + return [ + f"alpha: the document says {field} {expression!r}, the build recorded " + f"{recorded!r}" + for field in ("licenseConcluded", "licenseDeclared") + ] + + +@pytest.mark.parametrize( + "rewritten", ["LicenseRef-alpha", "Apache-2.0 AND LicenseRef-alpha"] +) +def test_the_named_half_of_a_mixed_conclusion_is_checked(eula_beside_mit, rewritten): + assert _rewrite_alpha(eula_beside_mit, rewritten) == _disagreement(rewritten) + + +def test_another_component_s_licence_ref_is_not_accepted(eula_beside_mit): + """Swapped, alpha's EULA would be published as the SDK's terms; every + extracted text would still belong to some component.""" + other = next( + item["licenseId"] + for item in eula_beside_mit["spdx"]["hasExtractedLicensingInfos"] + if "demosdk" in item["licenseId"] + ) + rewritten = f"{other} AND MIT" + + assert _rewrite_alpha(eula_beside_mit, rewritten) == _disagreement(rewritten) + + def test_reuse_tags_on_shipped_files_are_reported_per_file(built): spdx = _embedded(built["wheel"]) tagged = { @@ -256,32 +346,33 @@ def test_source_date_epoch_normalizes_timestamps_the_input_wheel_carried( assert len(stamps) == 1, f"members carry differing timestamps: {stamps}" -def test_components_sharing_a_licence_text_share_one_entry(workspace, license_data): - """A LicenseRef identifies a text, so one text is one entry. - - Two components can carry byte-identical terms -- two builds of one SDK, a - vendor licence applied across products -- and minting an id each duplicates - the whole text in the document. - """ +@pytest.fixture +def shared_text(workspace, license_data): + """alpha and beta ship the demo SDK's proprietary terms byte for byte.""" from sbom import licensing licensing.load_corpus(license_data) for key in ("alpha", "beta"): root = workspace.build / "_deps" / f"{key}-src" root.mkdir(parents=True, exist_ok=True) - (root / "LICENSE").write_text(synth.PROPRIETARY_LICENSE) + (root / "LICENSE").write_text(synth.PROPRIETARY_LICENSE, encoding="utf-8") synth.track(workspace.root) payload = synth.wheel_payload(workspace) for key in ("alpha", "beta"): payload[f"isaaccapture/{key}_notice.txt"] = synth.PROPRIETARY_LICENSE.encode() synth.write_wheel(workspace.wheel, payload) - build_module.build( - workspace.root, workspace.build, workspace.wheel, workspace.root / "sbom" - ) + return synth.collect_and_bind(workspace) + + +def test_components_sharing_a_licence_text_share_one_entry(shared_text): + """A LicenseRef identifies a text, so one text is one entry. - with zipfile.ZipFile(workspace.wheel) as archive: - spdx = json.loads(archive.read(f"{DIST_INFO}/sboms/{WHEEL_NAME}.spdx.json")) + Two components can carry byte-identical terms -- two builds of one SDK, a + vendor licence applied across products -- and minting an id each duplicates + the whole text in the document. + """ + spdx = _embedded(shared_text["wheel"]) bodies = [item["extractedText"] for item in spdx["hasExtractedLicensingInfos"]] assert len(bodies) == len(set(bodies)), "a text is carried more than once" @@ -295,6 +386,29 @@ def test_components_sharing_a_licence_text_share_one_entry(workspace, license_da assert used <= defined, f"undefined after sharing: {sorted(used - defined)}" +def test_a_shared_id_is_held_to_the_text_it_names(shared_text): + """gamma's Zlib text is some component's terms, so only the id catches it.""" + member = f"{DIST_INFO}/sboms/{WHEEL_NAME}.spdx.json" + payload = synth.members(shared_text["wheel"]) + spdx = json.loads(payload[member]) + entry = next( + item + for item in spdx["hasExtractedLicensingInfos"] + if item["licenseId"].startswith("LicenseRef-shared-text-") + ) + entry["extractedText"] = synth.spdx_text("Zlib") + payload[member] = json.dumps(spdx).encode() + synth.write_wheel(shared_text["wheel"], payload) + + failures = validate_module.check( + shared_text["wheel"], build_evidence=shared_text["evidence_path"] + ) + + assert failures == [ + f"{entry['licenseId']}: the text it carries is not the one its id names" + ] + + def test_a_requirement_this_run_also_built_is_not_an_unknown( workspace, license_data, tmp_path ): From 519bbe63fec0cad4536c9959b5f72024c8483f22 Mon Sep 17 00:00:00 2001 From: Andrei Aristarkhov Date: Tue, 20 Oct 2026 02:19:38 -0700 Subject: [PATCH 082/101] build(sbom): never give a fetched tree the enclosing repository's identity git was asked for the remote and describe of every _deps/-src, and a tree fetched as an archive inside this checkout answered with the checkout's own: OAK's SDL2 would ship as NVIDIA/IsaacCapture. git is now asked only of a directory that is its own repository, with git's repository-selecting environment stripped. Any other tree is identified by what the build declared, read from FetchContent's stamps in both population modes (the tag from the clone script they name): a GitHub archive by its repository and ref, any other URL as a generic package with no supplier. Published locations drop credentials, query and fragment. Signed-off-by: Andrei Aristarkhov --- docs/source/references/sbom.rst | 17 +- scripts/sbom/sbom/cli.py | 16 +- scripts/sbom/sbom/discovery.py | 330 +++++++++++++++++----- scripts/sbom/sbom/evidence.py | 6 +- scripts/sbom/sbom/licensing.py | 4 +- tests/python/sbom/conftest.py | 8 +- tests/python/sbom/synth.py | 125 ++++++-- tests/python/sbom/test_discovery.py | 377 ++++++++++++++++++++++++- tests/python/sbom/test_nested_terms.py | 4 +- 9 files changed, 757 insertions(+), 130 deletions(-) diff --git a/docs/source/references/sbom.rst b/docs/source/references/sbom.rst index 04386b217b..69bffb7456 100644 --- a/docs/source/references/sbom.rst +++ b/docs/source/references/sbom.rst @@ -143,8 +143,21 @@ There is no checked-in dependency list. Every entry is derived from the build th produced the wheel, so a dependency that builds is a dependency that appears: ``build/_deps/*-src`` - FetchContent checkouts. ``git`` in each one supplies the commit, the remote, and from - the remote the supplier, homepage and purl. + FetchContent trees. ``git`` in each checkout supplies the commit, the remote, and from + the remote the supplier, homepage and purl. It is asked only of a tree that is a + repository of its own: git otherwise answers for whichever checkout encloses the build + directory, or for whatever repository ``GIT_DIR`` names, which is therefore cleared. Any + other tree is identified by its declaration — ``GIT_REPOSITORY`` and ``GIT_TAG``, or + ``URL`` and ``URL_HASH`` — from the stamps ExternalProject writes beside the populate + sub-build, or under ``CMakeFiles/fc-stamp`` once policy ``CMP0168`` is ``NEW``, and + from the clone script the git stamp names. A stamp outlives a change of method, so where + both are present the tree decides: a git clone is a checkout of its own. The version is + the declared tag, or the ref a GitHub URL names, verbatim, so a tag gives one version + whether declared by tag or as a GitHub archive; any other archive's version is read + from its name, and its supplier is not asserted, since the host serving a download is + not who supplies it. A download location is stated only for an ``http``, ``https``, + ``ftp`` or ``ftps`` archive URL, or a git remote reached over ``http``, ``https``, + ``ssh`` or ``git``, and never with its credentials, query or fragment. vcpkg's install records Where the build uses vcpkg, the ``vcpkg/info`` lists under ``VCPKG_INSTALLED_DIR`` (by diff --git a/scripts/sbom/sbom/cli.py b/scripts/sbom/sbom/cli.py index 8ceb1ae04d..55a48ef08d 100644 --- a/scripts/sbom/sbom/cli.py +++ b/scripts/sbom/sbom/cli.py @@ -7,7 +7,6 @@ import argparse import json -import subprocess import sys import zipfile from pathlib import Path @@ -31,19 +30,8 @@ def _repo_root(value: str | None) -> Path: """ if value: return Path(value).resolve() - try: - result = subprocess.run( - ["git", "rev-parse", "--show-toplevel"], - capture_output=True, - text=True, - check=False, - timeout=30, - ) - except (OSError, subprocess.SubprocessError): - return Path.cwd() - return ( - Path(result.stdout.strip()).resolve() if result.returncode == 0 else Path.cwd() - ) + top = discovery.git(Path.cwd(), "rev-parse", "--show-toplevel") + return Path(top).resolve() if top else Path.cwd() def _report(failures: list[str], subject: str) -> int: diff --git a/scripts/sbom/sbom/discovery.py b/scripts/sbom/sbom/discovery.py index 2efeb0d916..e53e28a1ac 100644 --- a/scripts/sbom/sbom/discovery.py +++ b/scripts/sbom/sbom/discovery.py @@ -9,7 +9,7 @@ Four indexes do the work: -* source trees ``_deps/*-src`` checkouts -> identity, from git +* source trees ``_deps/*-src`` trees -> identity, from git or the declaration * build graph CMake's file API codemodel -> what links and compiles into what * archives members of tarballs in the repo, by content hash * files repo and build-tree files, by content hash @@ -34,6 +34,7 @@ from dataclasses import dataclass, field, replace from pathlib import Path, PurePosixPath from typing import NamedTuple +from urllib.parse import urlparse from packageurl import PackageURL @@ -59,7 +60,29 @@ _GITHUB_REMOTE = re.compile( r"github\.com[:/](?P[^/]+)/(?P[^/]+?)(?:\.git)?/?$" ) -_GITCLONE_URL = re.compile(r'clone\b[^\n]*?"(?P[a-z]+[^"\s]*://[^"]+|git@[^"]+)"') +# A release asset or a source archive on GitHub names its repository and its ref. +_GITHUB_REPO = r"(?P[^/]+)/(?P[^/]+)/" +_GITHUB_DOWNLOADS = ( + re.compile( + rf"^https?://github\.com/{_GITHUB_REPO}releases/download/(?P[^/]+)/[^/]+$" + ), + re.compile( + rf"^https?://github\.com/{_GITHUB_REPO}archive/(?:refs/(?:tags|heads)/)?" + rf"(?P.+?)(?:{'|'.join(map(re.escape, ARCHIVE_SUFFIXES))})$" + ), + re.compile( + rf"^https?://codeload\.github\.com/{_GITHUB_REPO}(?:legacy\.)?(?:tar\.gz|zip)/" + r"(?:refs/(?:tags|heads)/)?(?P.+)$" + ), +) +# Schemes this collector states as a downloadLocation; a local path names only the builder. +_DOWNLOAD_SCHEMES = ("http", "https", "ftp", "ftps") +# Transports a published git location may name; a local path names only the builder. +_GIT_TRANSPORTS = ("http", "https", "ssh", "git") +# git's scp-like `[user@]host:path`, which SPDX knows only as `ssh://`. +_SCP_REMOTE = re.compile(r"^(?P[^@/:]+@)?(?P[^@/:]{2,}):(?!//)/?") +_VERSION_IN_NAME = re.compile(r"(\d+\.\d+[\w.+-]*?)(?=-|\.tar|\.tgz|\.zip|$)") +# The ref ExternalProject's clone script checks out; `clone --no-checkout` is not one. _GITCLONE_TAG = re.compile(r'(?[^"]+)"') _INCLUDE_FLAG = re.compile(r"-(?:I|isystem)\s*(\S+)") _DEPS_SOURCE = re.compile(r"_deps/(?P[A-Za-z0-9_.+-]+)-src(?:/|$)") @@ -139,22 +162,51 @@ class Component: unlicensed: tuple[tuple[str, str], ...] = () -def repo_head(directory: Path) -> str | None: - """The commit of the checkout rooted at `directory`, if it is one. +def own_git(directory: Path, *args: str) -> str | None: + """A read-only git query about the checkout rooted at `directory`. - `git -C rev-parse HEAD` answers for whichever repository encloses the - directory, so a dependency whose checkout carries no .git of its own would - otherwise be stamped with this project's commit. + git answers for whichever repository encloses a directory, and `build/` sits + inside this project's checkout, so a tree fetched as an archive would + otherwise be identified as this project. None unless git's toplevel is + `directory` itself. """ - top = git(directory, "rev-parse", "--show-toplevel") - if top is None or Path(top).resolve() != directory.resolve(): + if not _is_own_checkout(directory.resolve()): return None - return git(directory, "rev-parse", "HEAD") + return git(directory, *args) + + +@functools.cache +def _is_own_checkout(resolved: Path) -> bool: + """Cached per resolved path: one collector run does not create checkouts.""" + top = git(resolved, "rev-parse", "--show-toplevel") + return top is not None and Path(top).resolve() == resolved def git(repo: Path, *args: str) -> str | None: - """Run a read-only git query, or return None if git cannot answer.""" - return _ask("git", "-C", str(repo), *args).strip() or None + """Run a read-only git query about `repo`, or return None if git cannot answer. + + git exports GIT_DIR and its kin to the hooks it runs; with them set, git + answers for that repository whatever `-C` names, so they are dropped. + Configuration passed as GIT_CONFIG_PARAMETERS or GIT_CONFIG_COUNT, such as a + container's `safe.directory`, is kept, as git keeps it for a submodule; + GIT_CONFIG is not, since it replaces the repository's own configuration. + """ + local = set(_git_local_variables()) + env = {name: value for name, value in os.environ.items() if name not in local} + return _ask("git", "-C", str(repo), *args, env=env).strip() or None + + +@functools.cache +def _git_local_variables() -> tuple[str, ...]: + listed = _ask("git", "rev-parse", "--local-env-vars").split() + kept = {"GIT_CONFIG_PARAMETERS", "GIT_CONFIG_COUNT"} + return tuple(name for name in listed if name not in kept) + + +def version_in_name(name: str) -> str | None: + """The version an archive's file name states, as in `SDL2-2.32.10.tar.gz`.""" + match = _VERSION_IN_NAME.search(name) + return match.group(1) if match else None def notice_from(evidence) -> str: @@ -180,87 +232,208 @@ def notice_from(evidence) -> str: return "\n".join(licensing.fold_notices(notices)) if notices else "NOASSERTION" +class _Identity(NamedTuple): + name: str + supplier: str + homepage: str + purl: str + download_location: str + version: str + + +def _github_identity(org: str, repo: str, ref: str | None) -> tuple[str, str, str]: + """Supplier, homepage and purl of a GitHub repository at `ref`.""" + purl = PackageURL(type="github", namespace=org, name=repo, version=ref) + return f"Organization: {org}", f"https://github.com/{org}/{repo}", purl.to_string() + + def _identity( - remote: str | None, name: str, commit: str | None -) -> tuple[str, str, str, str]: - """Supplier, homepage, purl and download location, read off the remote URL.""" - if remote: - match = _GITHUB_REMOTE.search(remote) - if match: - org, repo = match.group("org"), match.group("repo") - purl = PackageURL( - type="github", namespace=org, name=repo, version=commit - ).to_string() - download = f"git+https://github.com/{org}/{repo}.git" + ( - f"@{commit}" if commit else "" - ) - return ( - f"Organization: {org}", - f"https://github.com/{org}/{repo}", - purl, - download, - ) - purl = f"pkg:generic/{name}" + (f"@{commit}" if commit else "") - return ( - "NOASSERTION", - remote, - purl, - (f"git+{remote}@{commit}" if commit else remote), + remote: str | None, name: str, commit: str | None, tag: str | None = None +) -> _Identity: + """A git source's identity, read off its remote URL, at the commit, else at + the declared tag.""" + ref = commit or tag + version = ref or "NOASSERTION" + match = _GITHUB_REMOTE.search(remote) if remote else None + if match: + org, repo = match.group("org"), match.group("repo") + download = f"git+https://github.com/{org}/{repo}.git" + ( + f"@{ref}" if ref else "" ) - return "NOASSERTION", "NOASSERTION", f"pkg:generic/{name}", "NOASSERTION" + return _Identity(repo, *_github_identity(org, repo, ref), download, version) + if remote: + purl = PackageURL(type="generic", name=name, version=ref).to_string() + location = _SCP_REMOTE.sub(r"ssh://\g\g/", remote) + if urlparse(location).scheme.lower() not in _GIT_TRANSPORTS: + download = "NOASSERTION" + else: + download = f"git+{location}@{ref}" if ref else location + return _Identity(name, "NOASSERTION", remote, purl, download, version) + return _Identity( + name, + "NOASSERTION", + "NOASSERTION", + f"pkg:generic/{name}", + "NOASSERTION", + version, + ) -def _declared_constraint(deps_dir: Path, name: str) -> dict[str, str | None]: - clone = ( - deps_dir - / f"{name}-subbuild" - / f"{name}-populate-prefix" - / "tmp" - / f"{name}-populate-gitclone.cmake" +def _archive_identity(url: str, digest: str | None, name: str) -> tuple[_Identity, str]: + """An archive's identity, read off the URL it was declared with, and how the + declaration reads in `sourceInfo`.""" + parsed = urlparse(url) + file_name = PurePosixPath(parsed.path).name + downloadable = parsed.scheme.lower() in _DOWNLOAD_SCHEMES + download = url if downloadable else "NOASSERTION" + detail = f"Declared archive: {url if downloadable else f'local file {file_name}'}" + detail += f" ({digest})." if digest else "." + + match = next(filter(None, (form.match(url) for form in _GITHUB_DOWNLOADS)), None) + if match: + org, repo, ref = match.group("org", "repo", "ref") + github = _github_identity(org, repo, ref) + return _Identity(repo, *github, download, ref), detail + + qualifiers = {"download_url": url} if downloadable else {} + if digest and "=" in digest: + algorithm, value = digest.split("=", 1) + algorithm = algorithm.lower().replace("_", "-") + qualifiers["checksum"] = f"{algorithm}:{value.lower()}" + version = version_in_name(file_name) + purl = PackageURL( + type="generic", name=name, version=version, qualifiers=qualifiers or None ) - if not clone.is_file(): - return {"url": None, "tag": None} - text = clone.read_text(encoding="utf-8", errors="replace") - url = _GITCLONE_URL.search(text) - tag = _GITCLONE_TAG.search(text) - return { - "url": url.group("url") if url else None, - "tag": tag.group("tag") if tag else None, - } + identity = _Identity( + name, + "NOASSERTION", + "NOASSERTION", + purl.to_string(), + download, + version or "NOASSERTION", + ) + return identity, detail + + +@dataclass(frozen=True) +class _Declaration: + """What a FetchContent declaration asked for.""" + + repository: str | None = None + tag: str | None = None + url: str | None = None + hash: str | None = None + + +def _stamps(deps_dir: Path) -> dict[str, dict[str, dict[str, str]]]: + """Each tree's FetchContent stamps, by method; sbom.rst says where from.""" + # Under the populate sub-build, or, with CMP0168 NEW, the build's own CMakeFiles. + roots = ( + (deps_dir, "*-subbuild/*-populate-prefix/src/*-populate-stamp"), + (deps_dir.parent / "CMakeFiles" / "fc-stamp", "*"), + ) + newest: dict[tuple[str, str], tuple[int, dict[str, str]]] = {} + for base, pattern in roots: + for method in ("url", "git"): + for stamp in base.glob(f"{pattern}/*-{method}info.txt"): + fields = {} + text = stamp.read_text(encoding="utf-8", errors="replace") + for line in text.splitlines(): + key, separator, value = line.partition("=") + if separator and not line.startswith("#"): + fields[key.strip()] = value.strip() + if not fields.get("source_dir"): + continue + # Both layouts can hold one after a policy change; the newer wins. + slot = (Path(fields["source_dir"]).name, method) + written = stamp.stat().st_mtime_ns + if slot not in newest or written > newest[slot][0]: + newest[slot] = (written, fields) + by_tree: dict[str, dict[str, dict[str, str]]] = {} + for (tree, method), (_, fields) in newest.items(): + by_tree.setdefault(tree, {})[method] = fields + return by_tree + + +def _declaration(deps_dir: Path, fields: dict[str, str]) -> _Declaration: + tag = None + # The git stamp's command runs the clone script, which names the ref; the + # stamp's paths are absolute, so a moved build tree is rebased onto this one. + built_in = Path(fields["source_dir"]).parent.parent + for word in fields.get("command", "").split(";"): + if not word.endswith("-gitclone.cmake"): + continue + script = Path(word) + if not script.is_file() and script.is_relative_to(built_in): + script = deps_dir.parent / script.relative_to(built_in) + if script.is_file(): + text = script.read_text(encoding="utf-8", errors="replace") + match = _GITCLONE_TAG.search(text) + tag = match.group("tag") if match else None + return _Declaration( + repository=fields.get("repository") or None, + tag=tag, + # A CMake list of mirrors, tried in order. + url=fields.get("url(s)", "").split(";")[0] or None, + hash=fields.get("hash") or None, + ) + + +def _public_url(url: str) -> str: + """`url` without credentials in its authority, and without the query and + fragment, where a signed download carries its token.""" + parsed = urlparse(url) + if "@" not in parsed.netloc and not parsed.query and not parsed.fragment: + return url + netloc = parsed.netloc.rpartition("@")[2] + return parsed._replace(netloc=netloc, query="", fragment="").geturl() def discover_source_trees(deps_dir: Path) -> dict[str, Component]: - """Every FetchContent checkout under the build's ``_deps``.""" + """Every FetchContent tree under the build's ``_deps``.""" components: dict[str, Component] = {} + stamps = _stamps(deps_dir) for root in sorted(deps_dir.glob("*-src")): if not root.is_dir(): continue key = root.name.removesuffix("-src") - declared = _declared_constraint(deps_dir, key) - commit = repo_head(root) - remote = git(root, "config", "--get", "remote.origin.url") or declared["url"] - describe = git(root, "describe", "--tags", "--always") - - supplier, homepage, purl, download = _identity(remote, key, commit) - evidence = licensing.discover_in_tree(key, root, root.name) - concluded, declared_expression = licensing.expression(evidence, key) + # A stamp outlives a switch of method, and an unchanged one keeps its old + # mtime, so the tree decides: a git clone is a checkout of its own. + methods = stamps.get(root.name, {}) + method = "git" if _is_own_checkout(root.resolve()) else "url" + fields = methods.get(method) or next(iter(methods.values()), None) + declared = _declaration(deps_dir, fields) if fields else _Declaration() + commit = own_git(root, "rev-parse", "HEAD") + remote = ( + own_git(root, "config", "--get", "remote.origin.url") or declared.repository + ) + remote = _public_url(remote) if remote else None + describe = own_git(root, "describe", "--tags", "--always") details = [f"Fetched into {root.name}."] - if declared["tag"]: - details.append(f"Declared constraint: {declared['tag']}.") + if remote or not declared.url: + identity = _identity(remote, key, commit, declared.tag) + if declared.tag: + details.append(f"Declared constraint: {declared.tag}.") + else: + identity, detail = _archive_identity( + _public_url(declared.url), declared.hash, key + ) + details.append(detail) if describe: details.append(f"git describe: {describe}.") - name_match = _GITHUB_REMOTE.search(remote) if remote else None + evidence = licensing.discover_in_tree(key, root, root.name) + concluded, declared_expression = licensing.expression(evidence, key) components[key] = Component( key=key, - name=name_match.group("repo") if name_match else key, + name=identity.name, kind="source-tree", - supplier=supplier, - homepage=homepage, - purl=purl, - version=commit or declared["tag"] or "NOASSERTION", - download_location=download, + supplier=identity.supplier, + homepage=identity.homepage, + purl=identity.purl, + version=identity.version, + download_location=identity.download_location, source_info=" ".join(details), root=root, copyright_text=notice_from(evidence), @@ -700,11 +873,16 @@ def _search_directories(toolchains: Path | None, build_dir: Path) -> dict[str, _ return found -def _ask(*command: str) -> str: +def _ask(*command: str, env: dict[str, str] | None = None) -> str: """A command's standard output, or empty where it cannot run or fails.""" try: result = subprocess.run( - list(command), capture_output=True, text=True, check=False, timeout=60 + list(command), + capture_output=True, + text=True, + check=False, + timeout=60, + env=env, ) except (OSError, subprocess.SubprocessError): return "" @@ -1735,7 +1913,9 @@ def _vcpkg_component( else (location, "", "") ) if remote and _GITHUB_REMOTE.search(remote): - supplier, homepage, purl, download = _identity(remote, port, ref or None) + identity = _identity(remote, port, ref or None) + supplier, homepage, purl = identity.supplier, identity.homepage, identity.purl + download = identity.download_location else: supplier, homepage, download = "NOASSERTION", "NOASSERTION", upstream purl = PackageURL(type="generic", name=port, version=version).to_string() diff --git a/scripts/sbom/sbom/evidence.py b/scripts/sbom/sbom/evidence.py index 7f8bdbadfd..b6800a478c 100644 --- a/scripts/sbom/sbom/evidence.py +++ b/scripts/sbom/sbom/evidence.py @@ -32,6 +32,7 @@ discover_vcpkg, notice_from, tracked_files, + version_in_name, EMPTY_SHA256, ) @@ -170,10 +171,7 @@ def _archive_version(archives: ArchiveIndex, display: str) -> str | None: stated = archives.archives.get(display, {}).get("version_text") if stated: return stated - match = re.search( - r"(\d+\.\d+[\w.+-]*?)(?=-|\.tar|\.tgz|\.zip|$)", Path(display).name - ) - return match.group(1) if match else None + return version_in_name(Path(display).name) def discover(repo_root: Path, build_dir: Path) -> Discovery: diff --git a/scripts/sbom/sbom/licensing.py b/scripts/sbom/sbom/licensing.py index 02499f1106..c5ae6882e8 100644 --- a/scripts/sbom/sbom/licensing.py +++ b/scripts/sbom/sbom/licensing.py @@ -324,9 +324,9 @@ def load_corpus(json_dir: Path) -> str: def _checkout_commit(source_dir: Path) -> str: """The commit deps/third_party pinned, read back from the checkout itself.""" - from .discovery import repo_head + from .discovery import own_git - return repo_head(source_dir) or "NOASSERTION" + return own_git(source_dir, "rev-parse", "HEAD") or "NOASSERTION" def corpus() -> tuple[str, dict[str, Reference]]: diff --git a/tests/python/sbom/conftest.py b/tests/python/sbom/conftest.py index 04c4ddaef4..64d9c01bec 100644 --- a/tests/python/sbom/conftest.py +++ b/tests/python/sbom/conftest.py @@ -10,7 +10,7 @@ import pytest import synth -from sbom import licensing +from sbom import discovery, licensing repo_root = synth.load( @@ -61,6 +61,12 @@ def _license_data_loaded(license_data): synth.set_license_data(license_data) +@pytest.fixture(autouse=True) +def _fresh_checkout_answers(): + """Each test builds its own checkouts; a cached answer is a previous test's.""" + discovery._is_own_checkout.cache_clear() + + @pytest.fixture(scope="session") def checkout(): """This repository, as the collector would be pointed at it.""" diff --git a/tests/python/sbom/synth.py b/tests/python/sbom/synth.py index 9a6aec272f..f466947aa2 100644 --- a/tests/python/sbom/synth.py +++ b/tests/python/sbom/synth.py @@ -306,15 +306,8 @@ def __init__(self, root: Path) -> None: self.wheel = root / "dist" / WHEEL_NAME -def _git_checkout(root: Path, initialise: bool = True) -> None: - """Make the generated project a real checkout. - - The collector separates repository source from build output by what git - tracks, because an install prefix carries no marker saying it is output. - `build/` and `dist/` are ignored here for the same reason they are in the - real repository. - """ - write(root / ".gitignore", "build/\ndist/\n") +def git(root: Path, *args: str) -> str: + """Run git in `root` and return its output.""" # Fixed identity *and* clock: a commit hashes its timestamp, the collector # records the commit, and the wheel carries that record -- so a drifting # clock makes two builds of the same project differ. @@ -327,13 +320,50 @@ def _git_checkout(root: Path, initialise: bool = True) -> None: "GIT_COMMITTER_EMAIL": "synth@example.invalid", "GIT_COMMITTER_DATE": "2026-01-01T00:00:00+00:00", } - run = lambda *args: subprocess.run( # noqa: E731 - local shorthand - ["git", "-C", str(root), *args], check=True, capture_output=True, env=env - ) + return subprocess.run( + ["git", "-C", str(root), *args], + check=True, + capture_output=True, + env=env, + text=True, + ).stdout.strip() + + +def _commit_all(root: Path, message: str) -> None: + git(root, "add", "-A") + git(root, "commit", "-q", "--allow-empty", "-m", message) + + +def clone_of(root: Path, remote: str, tag: str, git_dir: Path | None = None) -> str: + """Make `root` a checkout of its own, as FetchContent's clone leaves it. + + `git_dir` puts the repository elsewhere behind a `.git` file, the way a + submodule's is. Returns the commit. + """ + root.mkdir(parents=True, exist_ok=True) + separate = [] + if git_dir: + git_dir.parent.mkdir(parents=True, exist_ok=True) + separate = ["--separate-git-dir", str(git_dir)] + git(root, "init", "-q", "-b", "main", *separate) + _commit_all(root, "upstream") + git(root, "tag", tag) + git(root, "remote", "add", "origin", remote) + return git(root, "rev-parse", "HEAD") + + +def _git_checkout(root: Path, initialise: bool = True) -> None: + """Make the generated project a real checkout. + + The collector separates repository source from build output by what git + tracks, because an install prefix carries no marker saying it is output. + `build/` and `dist/` are ignored here for the same reason they are in the + real repository. + """ + write(root / ".gitignore", "build/\ndist/\n") if initialise: - run("init", "-q", "-b", "main") - run("add", "-A") - run("commit", "-q", "--allow-empty", "-m", "synthetic project") + git(root, "init", "-q", "-b", "main") + _commit_all(root, "synthetic project") def annotation( @@ -384,15 +414,60 @@ def _stage_license_data(target: Path) -> None: write(target / "exceptions" / detail.name, detail.read_text(encoding="utf-8")) -def _gitclone_stub(deps: Path, key: str, remote: str) -> None: - tmp = deps / f"{key}-subbuild" / f"{key}-populate-prefix" / "tmp" - tmp.mkdir(parents=True, exist_ok=True) - write( - tmp / f"{key}-populate-gitclone.cmake", - " clone --no-checkout --depth 1 --no-single-branch " - f'--config "advice.detachedHead=false" "{remote}" "{key}-src"\n' - ' checkout "v1.2.3" --\n', - ) +def declare( + deps: Path, + key: str, + *, + urls: tuple[str, ...] = (), + digest: str | None = None, + repository: str | None = None, + tag: str | None = None, + direct: bool = False, + source: str | None = None, +) -> None: + """Write the stamps a FetchContent declaration leaves in the build tree. + + Laid out as the populate sub-build does, or, with `direct`, as CMP0168 NEW + does. `source` names the tree, `-src` by default. + """ + source_dir = deps / (source or f"{key}-src") + if direct: + files = deps.parent / "CMakeFiles" + stamps, scripts, prefix = files / "fc-stamp" / key, files / "fc-tmp" / key, key + else: + prefix = f"{key}-populate" + scripts = deps / f"{key}-subbuild" / f"{prefix}-prefix" / "tmp" + stamps = scripts.parent / "src" / f"{prefix}-stamp" + + def stamp(kind: str, **fields: str) -> None: + body = "".join(f"{name}={value}\n" for name, value in fields.items()) + write( + stamps / f"{prefix}-{kind}info.txt", + "# This is a generated file and its contents are an internal " + f"implementation detail.\n\nmethod={kind}\n{body}", + ) + + if urls: + stamp( + "url", + source_dir=str(source_dir), + **{"url(s)": ";".join(urls)}, + hash=digest or "", + ) + if repository: + clone = scripts / f"{prefix}-gitclone.cmake" + write( + clone, + # The URL straight after `--no-checkout`, which is not a checkout. + f' clone --no-checkout "{repository}" "{source_dir.name}"\n' + + (f' checkout "{tag}" --\n' if tag else ""), + ) + stamp( + "git", + command=f"/usr/bin/cmake;-P;{clone}", + source_dir=str(source_dir), + repository=repository, + ) def _sdk_tarball(path: Path) -> dict[str, bytes]: @@ -444,7 +519,7 @@ def create(root: Path) -> Workspace: source = deps / f"{key}-src" write(source / license_name, spdx_text(license_id)) write(source / f"{key}.cpp", f"// {key}\n") - _gitclone_stub(deps, key, remote) + declare(deps, key, repository=remote, tag="v1.2.3") # vendorpy ships Python that the build stages into the wheel. write(deps / "vendorpy-src/pkg/mod.py", _VENDOR_MODULE) diff --git a/tests/python/sbom/test_discovery.py b/tests/python/sbom/test_discovery.py index 4a68be1c1f..eff0dc58f6 100644 --- a/tests/python/sbom/test_discovery.py +++ b/tests/python/sbom/test_discovery.py @@ -6,6 +6,7 @@ from __future__ import annotations import json +import os import subprocess import zlib from pathlib import Path @@ -36,15 +37,379 @@ def test_identity_comes_off_the_remote_url(workspace): assert alpha.supplier == "Organization: example-org" assert alpha.homepage == "https://github.com/example-org/alpha" - assert alpha.purl.startswith("pkg:github/example-org/alpha") - assert alpha.version == "v1.2.3", "the declared constraint stands in for a commit" + # With no checkout, the declared tag stands in for a commit throughout. + assert alpha.purl == "pkg:github/example-org/alpha@v1.2.3" + assert ( + alpha.download_location == "git+https://github.com/example-org/alpha.git@v1.2.3" + ) + assert alpha.version == "v1.2.3" + + +SDL_RELEASE = ( + "https://github.com/libsdl-org/SDL/releases/download/" + "release-2.32.10/SDL2-2.32.10.tar.gz" +) +SDL_HASH = "SHA256=" + "5f" * 32 + + +@pytest.mark.parametrize("hook", [False, True], ids=["shell", "under-a-hook"]) +def test_an_archive_fetched_inside_a_checkout_is_not_that_checkout( + workspace, monkeypatch, hook +): + """git answers for the repository enclosing `build/`; the declaration does not. + + A hook runs with GIT_DIR exported, which makes every directory look like + the root of that repository. + """ + deps = workspace.build / "_deps" + synth.write(deps / "sdl2-src/LICENSE.txt", synth.spdx_text("Zlib")) + synth.declare(deps, "sdl2", urls=(SDL_RELEASE,), digest=SDL_HASH) + synth.git( + workspace.root, + "remote", + "add", + "origin", + "https://github.com/example/enclosing-project.git", + ) + if hook: + monkeypatch.setenv("GIT_DIR", str(workspace.root / ".git")) + + components = discovery.discover_source_trees(deps) + sdl2 = components["sdl2"] + + assert (sdl2.name, sdl2.supplier, sdl2.homepage, sdl2.purl, sdl2.version) == ( + "SDL", + "Organization: libsdl-org", + "https://github.com/libsdl-org/SDL", + "pkg:github/libsdl-org/sdl@release-2.32.10", + "release-2.32.10", + ) + assert sdl2.download_location == SDL_RELEASE + assert f"Declared archive: {SDL_RELEASE} ({SDL_HASH})." in sdl2.source_info + # A tree declared by GIT_REPOSITORY with no .git left takes no describe either. + for component in components.values(): + assert "enclosing-project" not in repr(component) + assert "git describe" not in component.source_info + + +def test_the_checkout_itself_is_read_whatever_git_dir_says( + workspace, tmp_path, monkeypatch +): + head = discovery.git(workspace.root, "rev-parse", "HEAD") + synth.clone_of(tmp_path / "other", "https://example.invalid/other.git", "v0") + monkeypatch.setenv("GIT_DIR", str(tmp_path / "other" / ".git")) + + assert discovery.git(workspace.root, "rev-parse", "HEAD") == head + assert "VERSION" in discovery.tracked_files(workspace.root) + + +DIRECT = pytest.mark.parametrize("direct", [False, True], ids=["sub-build", "direct"]) + + +@DIRECT +@pytest.mark.parametrize( + ("urls", "digest", "purl", "version", "downloadable"), + [ + ( + ("https://downloads.example.org/foo-1.4.2.tar.gz",), + "SHA256=AB12", + "pkg:generic/foo@1.4.2?checksum=sha256:ab12" + "&download_url=https://downloads.example.org/foo-1.4.2.tar.gz", + "1.4.2", + True, + ), + ( + ( + "https://mirror-a.example.org/foo-1.4.2.tar.gz", + "https://mirror-b.example.org/foo-1.4.3.tar.gz", + ), + "SHA3_256=CD34", + "pkg:generic/foo@1.4.2?checksum=sha3-256:cd34" + "&download_url=https://mirror-a.example.org/foo-1.4.2.tar.gz", + "1.4.2", + True, + ), + ( + ("https://gitlab.com/example-org/foo/-/archive/v1.5.0/foo-v1.5.0.tar.gz",), + None, + "pkg:generic/foo@1.5.0?download_url=" + "https://gitlab.com/example-org/foo/-/archive/v1.5.0/foo-v1.5.0.tar.gz", + "1.5.0", + True, + ), + ( + ("https://downloads.sourceforge.net/project/foo/foo-2.3.tar.bz2",), + None, + "pkg:generic/foo@2.3" + "?download_url=https://downloads.sourceforge.net/project/foo/foo-2.3.tar.bz2", + "2.3", + True, + ), + ( + ("/home/builder/archives/foo-1.4.2.tar.gz",), + None, + "pkg:generic/foo@1.4.2", + "1.4.2", + False, + ), + ( + ("file:///home/builder/archives/foo.zip",), + None, + "pkg:generic/foo", + "NOASSERTION", + False, + ), + ( + ("https://github.com/example-org/foo/archive/refs/tags/v3.0.tar.gz",), + None, + "pkg:github/example-org/foo@v3.0", + "v3.0", + True, + ), + ( + ("https://github.com/example-org/foo/archive/0a1b2c3d.zip",), + None, + "pkg:github/example-org/foo@0a1b2c3d", + "0a1b2c3d", + True, + ), + ( + ( + "https://github.com/example-org/foo/releases/download/v1.0/foo-linux.zip", + ), + None, + "pkg:github/example-org/foo@v1.0", + "v1.0", + True, + ), + ( + ( + "https://github.com/example-org/foo/archive/refs/tags/" + "releases/cpp/v2.1.1.tar.gz", + ), + None, + "pkg:github/example-org/foo@releases/cpp/v2.1.1", + "releases/cpp/v2.1.1", + True, + ), + ( + ("https://codeload.github.com/example-org/foo/tar.gz/refs/tags/v4.2",), + None, + "pkg:github/example-org/foo@v4.2", + "v4.2", + True, + ), + ], +) +def test_a_declared_archive_is_named_by_its_url( + tmp_path, urls, digest, purl, version, downloadable, direct +): + deps = tmp_path / "build" / "_deps" + (deps / "foo-src").mkdir(parents=True) + synth.declare(deps, "foo", urls=urls, digest=digest, direct=direct) + + foo = discovery.discover_source_trees(deps)["foo"] + + download = urls[0] if downloadable else "NOASSERTION" + assert (foo.purl, foo.version, foo.download_location) == (purl, version, download) + github = purl.startswith("pkg:github/") + assert foo.supplier == ("Organization: example-org" if github else "NOASSERTION") + assert "/home/builder" not in foo.source_info + + +@DIRECT +def test_a_declaration_is_matched_to_the_tree_it_names(tmp_path, direct): + """`license_list_data` fills `license-list-data-src`: the content name is not the tree.""" + deps = tmp_path / "build" / "_deps" + (deps / "foo-bar-src").mkdir(parents=True) + synth.declare( + deps, + "foo_bar", + repository="https://github.com/example-org/foo-bar.git", + tag="v5", + source="foo-bar-src", + direct=direct, + ) + foo_bar = discovery.discover_source_trees(deps)["foo-bar"] + + assert (foo_bar.purl, foo_bar.version) == ( + "pkg:github/example-org/foo-bar@v5", + "v5", + ) + assert "Declared constraint: v5." in foo_bar.source_info + + +@pytest.mark.parametrize("stale_is_newer", [False, True], ids=["older", "newer"]) +def test_a_stamp_left_by_an_earlier_declaration_does_not_count( + tmp_path, stale_is_newer +): + """Switching method leaves the other stamp behind, and CMake rewrites a + stamp only when its contents change, so its mtime cannot decide.""" + deps = tmp_path / "build" / "_deps" + (deps / "foo-src").mkdir(parents=True) + synth.declare(deps, "foo", repository="https://example.invalid/foo.git", tag="v1") + synth.declare(deps, "foo", urls=("https://downloads.example.org/foo-2.0.tar.gz",)) + git_stamp = next(deps.glob("foo-subbuild/**/*-gitinfo.txt")) + url_stamp = next(deps.glob("foo-subbuild/**/*-urlinfo.txt")) + stale, current = (git_stamp, url_stamp) + os.utime(stale, ns=(2, 2) if stale_is_newer else (0, 0)) + os.utime(current, ns=(1, 1)) + + foo = discovery.discover_source_trees(deps)["foo"] + + assert (foo.version, foo.purl.split("?")[0]) == ("2.0", "pkg:generic/foo@2.0") + assert "example.invalid" not in repr(foo) + + +def test_after_a_policy_change_the_newer_layout_is_the_declaration(tmp_path): + """Sub-build stamps stay behind when CMP0168 turns NEW, and the reverse.""" + deps = tmp_path / "build" / "_deps" + (deps / "foo-src").mkdir(parents=True) + synth.declare(deps, "foo", urls=("https://downloads.example.org/foo-1.0.tar.gz",)) + synth.declare( + deps, "foo", urls=("https://downloads.example.org/foo-2.0.tar.gz",), direct=True + ) + os.utime(next(deps.glob("foo-subbuild/**/*-urlinfo.txt")), ns=(0, 0)) + + assert discovery.discover_source_trees(deps)["foo"].version == "2.0" + + +def test_a_checkout_takes_its_git_stamp_over_a_newer_url_stamp(tmp_path): + deps = tmp_path / "build" / "_deps" + commit = synth.clone_of(deps / "foo-src", "https://git.example.org/foo.git", "v3") + synth.declare(deps, "foo", repository="https://git.example.org/foo.git", tag="v3") + synth.declare(deps, "foo", urls=("https://downloads.example.org/foo-2.0.tar.gz",)) + os.utime(next(deps.glob("foo-subbuild/**/*-gitinfo.txt")), ns=(0, 0)) + + foo = discovery.discover_source_trees(deps)["foo"] + + assert foo.version == commit + assert "Declared constraint: v3." in foo.source_info + assert "downloads.example.org" not in repr(foo) + + +def test_git_config_in_the_environment_does_not_choose_the_remote( + tmp_path, monkeypatch +): + deps = tmp_path / "build" / "_deps" + synth.clone_of(deps / "foo-src", "https://git.example.org/foo.git", "v1") + synth.clone_of(tmp_path / "other", "https://git.example.org/other.git", "v1") + monkeypatch.setenv("GIT_CONFIG", str(tmp_path / "other" / ".git" / "config")) + + foo = discovery.discover_source_trees(deps)["foo"] + + assert foo.homepage == "https://git.example.org/foo.git" + + +@pytest.mark.parametrize( + ("repository", "tag", "purl", "download"), + [ + ( + "https://git.example.org/foo.git", + "v1", + "pkg:generic/foo@v1", + "git+https://git.example.org/foo.git@v1", + ), + ( + "https://git.example.org/foo.git", + "v1.0+build", + "pkg:generic/foo@v1.0%2Bbuild", + "git+https://git.example.org/foo.git@v1.0+build", + ), + ( + "git@git.example.org:org/foo.git", + "v1", + "pkg:generic/foo@v1", + "git+ssh://git@git.example.org/org/foo.git@v1", + ), + ("/srv/git/foo.git", "v1", "pkg:generic/foo@v1", "NOASSERTION"), + ], +) +def test_a_declared_repository_off_github_is_named_by_its_tag( + tmp_path, repository, tag, purl, download +): + deps = tmp_path / "build" / "_deps" + (deps / "foo-src").mkdir(parents=True) + synth.declare(deps, "foo", repository=repository, tag=tag) + + foo = discovery.discover_source_trees(deps)["foo"] + + assert (foo.purl, foo.download_location, foo.version) == (purl, download, tag) + + +def test_a_moved_build_tree_still_reads_its_clone_script(tmp_path): + deps = tmp_path / "old" / "_deps" + (deps / "foo-src").mkdir(parents=True) + synth.declare(deps, "foo", repository="https://example.invalid/foo.git", tag="v7") + (tmp_path / "old").rename(tmp_path / "new") + + foo = discovery.discover_source_trees(tmp_path / "new" / "_deps")["foo"] + + assert foo.version == "v7" + + +def test_credentials_in_a_declared_url_are_not_published(tmp_path): + deps = tmp_path / "build" / "_deps" + (deps / "foo-src").mkdir(parents=True) + (deps / "bar-src").mkdir(parents=True) + synth.declare( + deps, + "foo", + urls=("https://u:s3cret@dl.example.org/foo-1.0.tgz?token=t0ken#sig=s1g",), + ) + synth.declare(deps, "bar", repository="https://tok3n@git.example.org/bar.git") + + components = discovery.discover_source_trees(deps) + + for secret in ("s3cret", "t0ken", "s1g"): + assert secret not in repr(components["foo"]) + assert "tok3n" not in repr(components["bar"]) + assert components["foo"].download_location == "https://dl.example.org/foo-1.0.tgz" + assert components["foo"].purl.endswith( + "download_url=https://dl.example.org/foo-1.0.tgz" + ) + + +def test_configuration_from_the_environment_reaches_git(workspace, monkeypatch): + """A container's safe.directory arrives this way; it does not pick a repository.""" + monkeypatch.setenv("GIT_CONFIG_COUNT", "1") + monkeypatch.setenv("GIT_CONFIG_KEY_0", "sbom.probe") + monkeypatch.setenv("GIT_CONFIG_VALUE_0", "kept") + + assert discovery.git(workspace.root, "config", "--get", "sbom.probe") == "kept" + + +@pytest.mark.parametrize( + ("in_another_checkout", "separate_git_dir", "through_symlink"), + [(False, False, False), (True, True, False), (False, False, True)], + ids=["standalone", "submodule-like", "symlinked"], +) +def test_a_checkout_of_its_own_answers_for_itself( + workspace, tmp_path, in_another_checkout, separate_git_dir, through_symlink +): + build = workspace.build if in_another_checkout else tmp_path / "build" + root = build / "_deps" / "epsilon-src" + synth.write(root / "LICENSE", synth.spdx_text("MIT")) + commit = synth.clone_of( + root, + "https://github.com/example-org/epsilon.git", + "v2.0.0", + git_dir=tmp_path / "modules" / "epsilon" if separate_git_dir else None, + ) + synth.declare( + build / "_deps", "epsilon", repository="https://example.invalid/x.git" + ) + if through_symlink: + (tmp_path / "link").symlink_to(build, target_is_directory=True) + build = tmp_path / "link" -def test_no_checkout_on_the_clone_line_is_not_a_checkout(workspace): - """`git clone --no-checkout ` must not be read as `git checkout `.""" - beta = discovery.discover_source_trees(workspace.build / "_deps")["beta"] + epsilon = discovery.discover_source_trees(build / "_deps")["epsilon"] - assert not beta.version.startswith("http") + assert epsilon.version == commit + assert epsilon.purl == f"pkg:github/example-org/epsilon@{commit}" + assert epsilon.download_location.endswith(f"/epsilon.git@{commit}") + assert "git describe: v2.0.0." in epsilon.source_info def test_licenses_come_from_the_trees_own_files(workspace): diff --git a/tests/python/sbom/test_nested_terms.py b/tests/python/sbom/test_nested_terms.py index 689e5bf2a6..bcd553b0c0 100644 --- a/tests/python/sbom/test_nested_terms.py +++ b/tests/python/sbom/test_nested_terms.py @@ -460,7 +460,9 @@ def _ninja(workspace, monkeypatch, log: str) -> None: monkeypatch.setattr( discovery, "_ask", - lambda *command: log if command[0] == "ninja" else asked(*command), + lambda *command, **options: ( + log if command[0] == "ninja" else asked(*command, **options) + ), ) From d9b3ae28b847e682bb2f1f88fa978fb9c05ce1d6 Mon Sep 17 00:00:00 2001 From: Andrei Aristarkhov Date: Tue, 20 Oct 2026 03:06:27 -0700 Subject: [PATCH 083/101] build(sbom): refuse a manifest that lists no wheel, and a wheel no manifest lists check-set iterated `manifest.get("wheels") or []`, so a manifest with no wheels checked nothing and printed OK; merge-manifests merged one. Every command reading a manifest now refuses one that is not an object, lists no wheel, or holds a record without a bare file name or with sidecars that name no file and digest, naming the file it read. check-set also fails a wheel in --wheel-dir the manifest does not list, and publish-wheel applies the same rule before it uploads, since every wheel there is published. Signed-off-by: Andrei Aristarkhov --- .github/workflows/build-ubuntu.yml | 20 +++- docs/source/references/sbom.rst | 12 +- scripts/sbom/sbom/build.py | 3 +- scripts/sbom/sbom/cli.py | 6 +- scripts/sbom/sbom/validate.py | 90 ++++++++++---- tests/python/sbom/test_gates.py | 185 +++++++++++++++++++++++++++++ 6 files changed, 285 insertions(+), 31 deletions(-) diff --git a/.github/workflows/build-ubuntu.yml b/.github/workflows/build-ubuntu.yml index 95170875ea..2ab293320f 100644 --- a/.github/workflows/build-ubuntu.yml +++ b/.github/workflows/build-ubuntu.yml @@ -1238,7 +1238,7 @@ jobs: - name: Ensure clean wheels directory run: | - rm -rf wheels + rm -rf wheels sbom-sidecars - name: Download wheel artifacts uses: actions/download-artifact@v7 @@ -1272,7 +1272,7 @@ jobs: exit 1 fi - published=() + declare -A named=() for manifest in "${manifests[@]}"; do while IFS=$'\t' read -r filename digest; do wheel="wheels/${filename}" @@ -1287,14 +1287,24 @@ jobs: echo " actual: ${actual}" exit 1 fi - published+=("${filename}") + named["${filename}"]=1 done < <(jq -r '.wheels[] | [.filename, .sha256] | @tsv' "${manifest}") done + # Every wheel under wheels/ is uploaded below, so one no manifest names + # would be published with no evidence checked for it. + for wheel in wheels/*.whl; do + name="$(basename "${wheel}")" + if [[ -z "${named[${name}]:-}" ]]; then + echo "${name} is among the downloaded wheels, and no manifest names it" + exit 1 + fi + done + for platform in "${platforms[@]}"; do for abi in "${abis[@]}"; do found=0 - for filename in "${published[@]}"; do + for filename in "${!named[@]}"; do if [[ "${filename}" == *"-${abi}-${abi}-manylinux"*"_${platform}.whl" ]]; then found=1 break @@ -1302,7 +1312,7 @@ jobs: done if (( found == 0 )); then echo "No wheel was published for ${abi} on ${platform}" - printf ' have: %s\n' "${published[@]}" + printf ' have: %s\n' "${!named[@]}" exit 1 fi done diff --git a/docs/source/references/sbom.rst b/docs/source/references/sbom.rst index 69bffb7456..f5dc7e5ac5 100644 --- a/docs/source/references/sbom.rst +++ b/docs/source/references/sbom.rst @@ -109,14 +109,18 @@ CI artifacts it travelled in: The wheel digest is bound here, after the last change to the wheel, so a wheel that was touched again no longer matches. -Downloading a whole release? Check the wheels and the evidence against the manifest in -one step: +Downloading a whole release? Put its wheels and evidence files in a directory of their +own, then check them against the manifest in one step: .. code-block:: bash uv run --project IsaacCapture/scripts/sbom isaaccapture-sbom check-set \ - --manifest isaaccapture-sbom-manifest.json \ - --wheel-dir . --evidence-dir . + --manifest release/isaaccapture-sbom-manifest.json \ + --wheel-dir release --evidence-dir release + +Any command reading a manifest refuses one that lists no wheel. ``check-set`` also fails a +wheel in ``--wheel-dir`` that the manifest does not list, which is why the directory holds +the release and nothing else. What is in scope ---------------- diff --git a/scripts/sbom/sbom/build.py b/scripts/sbom/sbom/build.py index ef0d9e3301..5878292b20 100644 --- a/scripts/sbom/sbom/build.py +++ b/scripts/sbom/sbom/build.py @@ -19,6 +19,7 @@ from . import licensing from . import inventory as inventory_module from . import report as report_module +from . import validate as validate_module from . import wheelfile from packaging.requirements import Requirement from packaging.utils import canonicalize_name @@ -581,7 +582,7 @@ def merge_manifests(paths: list[Path]) -> dict: [ item for path in sorted(paths) - for item in json.loads(path.read_text(encoding="utf-8"))["wheels"] + for item in validate_module.manifest_records(path) ] ) diff --git a/scripts/sbom/sbom/cli.py b/scripts/sbom/sbom/cli.py index 55a48ef08d..660930ffdb 100644 --- a/scripts/sbom/sbom/cli.py +++ b/scripts/sbom/sbom/cli.py @@ -183,7 +183,11 @@ def main(argv: list[str] | None = None) -> int: "check-set", help="run check over every wheel a merged manifest advertises" ) check_set.add_argument("--manifest", required=True) - check_set.add_argument("--wheel-dir", required=True) + check_set.add_argument( + "--wheel-dir", + required=True, + help="the release's wheels; one the manifest does not list fails the set", + ) check_set.add_argument("--evidence-dir", required=True) check_set.set_defaults(handler=_cmd_check_set) diff --git a/scripts/sbom/sbom/validate.py b/scripts/sbom/sbom/validate.py index 1dac3b8c70..ccc7fc7c1a 100644 --- a/scripts/sbom/sbom/validate.py +++ b/scripts/sbom/sbom/validate.py @@ -38,21 +38,66 @@ def _sidecar(evidence_dir: Path, record: dict, kind: str) -> Path | None: return path if path.is_file() else None -def _record_for(manifest: dict, filename: str) -> dict: - """One wheel's record, or a clear error rather than a KeyError traceback.""" +def manifest_records(path: Path) -> list[dict]: + """A manifest's wheel records; one listing none checks nothing, so it fails.""" try: - records = manifest["wheels"] - except (KeyError, TypeError) as error: - raise ManifestError("manifest has no 'wheels' list") from error + manifest = json.loads(path.read_text(encoding="utf-8")) + except ValueError as error: + raise ManifestError(f"{path}: the manifest is not JSON ({error})") from error + if not isinstance(manifest, dict): + raise ManifestError(f"{path}: the manifest is not a JSON object") + records = manifest.get("wheels") + if not isinstance(records, list): + raise ManifestError(f"{path}: the manifest has no 'wheels' list") + if not records: + raise ManifestError(f"{path}: the manifest lists no wheels") + for item in records: + if not isinstance(item, dict): + raise ManifestError(f"{path}: a 'wheels' entry is not an object") + if not isinstance(item.get("filename"), str) or not item["filename"]: + raise ManifestError(f"{path}: a 'wheels' entry names no wheel") + if not _is_bare_name(item["filename"]): + raise ManifestError( + f"{path}: {item['filename']!r} is not a bare wheel file name" + ) + return records + + +def _is_bare_name(name: str) -> bool: + """A file name joined onto a directory must stay inside that directory.""" + return name not in ("", ".", "..") and not any(sep in name for sep in "/\\:") + + +def _record_for(records: list[dict], filename: str) -> dict: + """One wheel's record, or a clear error rather than a KeyError traceback.""" seen = [item for item in records if item.get("filename") == filename] if len(seen) > 1: raise ManifestError(f"{filename} is advertised {len(seen)} times") - if not seen: - return {} - record = seen[0] + return _checked_record(seen[0]) if seen else {} + + +def _checked_record(record: dict) -> dict: + """A record holding every field the checks read, or ManifestError.""" + filename = record["filename"] for key in ("sha256", "size", "sbom_in_wheel", "sidecars"): if key not in record: raise ManifestError(f"{filename}: manifest record has no {key!r}") + sidecars = record["sidecars"] + if not isinstance(sidecars, dict) or not all( + isinstance(item, dict) + and isinstance(item.get("filename"), str) + and isinstance(item.get("sha256"), str) + for item in sidecars.values() + ): + raise ManifestError( + f"{filename}: manifest record's sidecars must each name a file and its sha256" + ) + for kind, item in sidecars.items(): + if not _is_bare_name(item["filename"]): + raise ManifestError( + f"{filename}: {kind} sidecar {item['filename']!r} is not a bare " + "file name" + ) # Every sidecar, not just the ones a record happens to name. An absent entry # is not an absent check: omitting `build_evidence` left `check-set` passing # a wheel the evidence would have failed. @@ -534,8 +579,7 @@ def _check_manifest( ) -> list[str]: """The manifest binds a wheel's digest and names the sidecars beside it.""" failures: list[str] = [] - manifest = json.loads(manifest_path.read_text(encoding="utf-8")) - record = _record_for(manifest, wheel.name) + record = _record_for(manifest_records(manifest_path), wheel.name) if not record: return [f"{manifest_path.name} has no entry for {wheel.name}"] @@ -628,15 +672,25 @@ def _cross_document_refs( def check_set(manifest_path: Path, wheel_dir: Path, evidence_dir: Path) -> list[str]: """Run :func:`check` over every wheel a merged manifest advertises.""" failures: list[str] = [] - manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + records = manifest_records(manifest_path) + for name, seen in sorted(Counter(item["filename"] for item in records).items()): + if seen > 1: + raise ManifestError(f"{name} is advertised {seen} times") + by_name = {item["filename"]: _checked_record(item) for item in records} + + # Every wheel here is published with the set, so one the manifest omits + # would go out unchecked. + for stray in sorted({item.name for item in wheel_dir.glob("*.whl")} - set(by_name)): + failures.append( + f"{stray} is in {wheel_dir}, and {manifest_path.name} does not list it" + ) # The whole release is here, which is the only place a reference from one # document to another can be resolved rather than merely declared. published: dict[str, bytes] = {} - for entry in manifest.get("wheels") or []: - record = _record_for(manifest, entry.get("filename") or "") - sidecar = _sidecar(evidence_dir, record, "spdx") if record else None - if sidecar and sidecar.is_file(): + for record in by_name.values(): + sidecar = _sidecar(evidence_dir, record, "spdx") + if sidecar: published[sidecar.name] = sidecar.read_bytes() for name, body in sorted(published.items()): @@ -645,11 +699,7 @@ def check_set(manifest_path: Path, wheel_dir: Path, evidence_dir: Path) -> list[ except json.JSONDecodeError as error: failures.append(f"{name}: not readable as a document ({error})") - for entry in manifest.get("wheels") or []: - named = entry.get("filename") - if not named: - raise ManifestError("a manifest record names no wheel") - record = _record_for(manifest, named) + for record in by_name.values(): wheel = wheel_dir / record["filename"] if not wheel.is_file(): failures.append(f"{record['filename']} is missing from {wheel_dir}") diff --git a/tests/python/sbom/test_gates.py b/tests/python/sbom/test_gates.py index 1f10eff4ba..73d2cfca5f 100644 --- a/tests/python/sbom/test_gates.py +++ b/tests/python/sbom/test_gates.py @@ -6,12 +6,14 @@ from __future__ import annotations import json +import re import shutil import zipfile import pytest import synth from sbom import build as build_module +from sbom import cli from sbom import licensing from sbom import wheelfile from sbom import validate as validate_module @@ -254,6 +256,7 @@ def test_a_complete_release_evidence_set_verifies(built, tmp_path): merged, wheels, evidence = _release_layout(built, tmp_path) assert validate_module.check_set(merged, wheels, evidence) == [] + assert _check_set_cli(merged, wheels, evidence) == 0 def test_a_missing_sidecar_fails_the_release_set(built, tmp_path): @@ -274,6 +277,188 @@ def test_a_missing_wheel_fails_the_release_set(built, tmp_path): assert any("is missing from" in failure for failure in failures) +def _check_set_cli(merged, wheels, evidence): + return cli.main( + [ + "check-set", + "--manifest", + str(merged), + "--wheel-dir", + str(wheels), + "--evidence-dir", + str(evidence), + ] + ) + + +@pytest.mark.parametrize( + ("manifest", "message"), + [ + ({}, "no 'wheels' list"), + ([], "not a JSON object"), + ({"wheels": []}, "lists no wheels"), + ({"wheels": "x"}, "no 'wheels' list"), + ({"wheels": ["x"]}, "entry is not an object"), + ({"wheels": [{}]}, "entry names no wheel"), + ({"wheels": [{"filename": 1}]}, "entry names no wheel"), + ({"wheels": [{"filename": "../x.whl"}]}, "not a bare wheel file name"), + ({"wheels": [{"filename": "/tmp/x.whl"}]}, "not a bare wheel file name"), + ({"wheels": [{"filename": "a\\b.whl"}]}, "not a bare wheel file name"), + ({"wheels": [{"filename": "C:x.whl"}]}, "not a bare wheel file name"), + ], +) +def test_manifest_records_refuses(tmp_path, manifest, message): + path = tmp_path / "manifest.json" + path.write_text(json.dumps(manifest), encoding="utf-8") + + with pytest.raises( + validate_module.ManifestError, match=f"{re.escape(str(path))}: .*{message}" + ): + validate_module.manifest_records(path) + + +def test_manifest_records_names_the_file_it_cannot_parse(tmp_path): + path = tmp_path / "manifest.json" + path.write_text("{", encoding="utf-8") + + with pytest.raises( + validate_module.ManifestError, match=rf"{re.escape(str(path))}: .*not JSON" + ): + validate_module.manifest_records(path) + + +def test_check_set_refuses_sidecars_that_are_not_named_files(built, tmp_path): + merged, wheels, evidence = _release_layout(built, tmp_path) + manifest = json.loads(merged.read_text(encoding="utf-8")) + manifest["wheels"][0]["sidecars"] = list(manifest["wheels"][0]["sidecars"]) + merged.write_text(json.dumps(manifest), encoding="utf-8") + + with pytest.raises( + validate_module.ManifestError, match="sidecars must each name a file" + ): + validate_module.check_set(merged, wheels, evidence) + + +def _manifest_of(tmp_path, records): + path = tmp_path / "manifest.json" + path.write_text(json.dumps({"wheels": records}), encoding="utf-8") + return path + + +def _record(filename="x-1.0-py3-none-any.whl", sidecar="x.spdx.json"): + return { + "filename": filename, + "sha256": "0" * 64, + "size": 0, + "sbom_in_wheel": "x-1.0.dist-info/sboms/x.spdx.json", + "sidecars": {"spdx": {"filename": sidecar, "sha256": "0" * 64}}, + } + + +@pytest.mark.parametrize("sidecar", ["../x.spdx.json", "/tmp/x.spdx.json", ".."]) +def test_check_set_refuses_a_sidecar_that_is_not_a_bare_name(tmp_path, sidecar): + manifest = _manifest_of(tmp_path, [_record(sidecar=sidecar)]) + + with pytest.raises(validate_module.ManifestError, match="not a bare file name"): + validate_module.check_set(manifest, tmp_path, tmp_path) + + +def test_check_set_refuses_a_sidecar_with_no_digest(tmp_path): + record = _record() + del record["sidecars"]["spdx"]["sha256"] + manifest = _manifest_of(tmp_path, [record]) + + with pytest.raises( + validate_module.ManifestError, match="sidecars must each name a file" + ): + validate_module.check_set(manifest, tmp_path, tmp_path) + + +def test_check_set_refuses_a_wheel_advertised_twice(tmp_path): + manifest = _manifest_of(tmp_path, [_record(), _record()]) + + with pytest.raises(validate_module.ManifestError, match="advertised 2 times"): + validate_module.check_set(manifest, tmp_path, tmp_path) + + +def _empty_manifest(tmp_path): + path = tmp_path / "manifest.json" + path.write_text(json.dumps({"wheels": []}), encoding="utf-8") + return path + + +def test_check_set_refuses_a_manifest_listing_no_wheel(tmp_path, capsys): + manifest = _empty_manifest(tmp_path) + + with pytest.raises(validate_module.ManifestError, match="lists no wheels"): + validate_module.check_set(manifest, tmp_path, tmp_path) + assert _check_set_cli(manifest, tmp_path, tmp_path) == 2 + assert "lists no wheels" in capsys.readouterr().err + + +def _tiny_wheel(tmp_path): + return synth.write_wheel( + tmp_path / "x-1.0-py3-none-any.whl", + { + "x-1.0.dist-info/METADATA": b"Metadata-Version: 2.4\nName: x\nVersion: 1.0\n", + "x-1.0.dist-info/WHEEL": b"Wheel-Version: 1.0\nRoot-Is-Purelib: true\n", + }, + ) + + +def test_check_refuses_a_manifest_listing_no_wheel(tmp_path): + with pytest.raises(validate_module.ManifestError, match="lists no wheels"): + validate_module.check(_tiny_wheel(tmp_path), manifest=_empty_manifest(tmp_path)) + + +@pytest.mark.parametrize( + ("records", "message"), + [ + ([_record(sidecar="../x.spdx.json")], "not a bare file name"), + ([_record(), _record()], "advertised 2 times"), + ], +) +def test_check_refuses_a_malformed_record_for_its_wheel(tmp_path, records, message): + manifest = _manifest_of(tmp_path, records) + + with pytest.raises(validate_module.ManifestError, match=message): + validate_module.check(_tiny_wheel(tmp_path), manifest=manifest) + + +def test_merge_manifests_refuses_a_manifest_listing_no_wheel(tmp_path): + good = tmp_path / "a" / "manifest.json" + good.parent.mkdir() + good.write_text( + json.dumps({"wheels": [{"filename": "x-1.0-py3-none-any.whl"}]}), + encoding="utf-8", + ) + (tmp_path / "b").mkdir() + empty = _empty_manifest(tmp_path / "b") + + with pytest.raises( + validate_module.ManifestError, match=f"{re.escape(str(empty))}: " + ): + build_module.merge_manifests([good, empty]) + output = str(tmp_path / "m.json") + assert cli.main(["merge-manifests", "--output", output, str(empty)]) == 2 + + +def test_a_wheel_the_manifest_does_not_list_fails_the_release_set( + built, tmp_path, capsys +): + merged, wheels, evidence = _release_layout(built, tmp_path) + stray = wheels / "isaacteleop-0.4.0-py3-none-any.whl" + stray.write_bytes(built["wheel"].read_bytes()) + + failures = validate_module.check_set(merged, wheels, evidence) + + assert failures == [ + f"{stray.name} is in {wheels}, and {merged.name} does not list it" + ] + assert _check_set_cli(merged, wheels, evidence) == 1 + assert "does not list it" in capsys.readouterr().err + + def test_missing_spdx_license_data_stops_the_build(workspace): """The reference texts come from the build; without them, nothing publishes.""" shutil.rmtree(workspace.build / "_deps/license-list-data-src") From 74ddf844c11cf769afe164c2a59006195ed65bd8 Mon Sep 17 00:00:00 2001 From: Andrei Aristarkhov Date: Tue, 20 Oct 2026 03:36:34 -0700 Subject: [PATCH 084/101] build(sbom): report a component whose licence nothing concludes A component whose texts were only a REUSE pool shipped those texts, so it escaped "no evidence", and concluded NOASSERTION, so it escaped the carried list; the report then said every component ships an identified license. A component concluding NOASSERTION is now listed under its own gap, with a reason naming its texts, and the summary line appears only when every gap list is empty. GAP_LISTS is the one source for the report's lists, the Markdown headings, the manifest and the CI warnings. The wheel's aggregate licence is NOASSERTION when any component it ships is, rather than leaving that component out. Signed-off-by: Andrei Aristarkhov --- docs/source/references/sbom.rst | 12 +++- scripts/sbom/sbom/build.py | 7 +- scripts/sbom/sbom/cli.py | 8 +-- scripts/sbom/sbom/document.py | 10 +-- scripts/sbom/sbom/licensing.py | 27 +++++++ scripts/sbom/sbom/report.py | 50 +++++++------ tests/python/sbom/test_licensing.py | 31 +++++++- tests/python/sbom/test_wheel_sbom.py | 102 ++++++++++++++++++++++----- 8 files changed, 191 insertions(+), 56 deletions(-) diff --git a/docs/source/references/sbom.rst b/docs/source/references/sbom.rst index f5dc7e5ac5..a71b9cda85 100644 --- a/docs/source/references/sbom.rst +++ b/docs/source/references/sbom.rst @@ -282,8 +282,10 @@ not say who licensed it to them. ``licenseDeclared`` on the wheel is what the distribution says of itself. ``licenseConcluded`` is what this document found in it, so a wheel carrying a component whose terms the build -could not name states that expression alongside the declaration — a reader asking "what is -this artifact" is told about the proprietary payload rather than about the declaration only. +could not name states that component's ``LicenseRef-`` alongside the declaration — a reader +asking "what is this artifact" is told about the proprietary payload rather than about the +declaration only. Where nothing at all is concluded for a component, neither is anything +for the wheel: it states ``NOASSERTION``. Never a file from this checkout. ``LICENSES/`` holds this repository's REUSE pool, whose copies exist to license *this* project and carry its notices; packaging one as a third @@ -314,6 +316,12 @@ fifths of, its copyright notices aside, beside the identifiers it names: the res terms no identifier names, as in fmt's MIT with its exception paragraph. The ``LicenseRef-`` carries exactly the texts carried. Those components are listed in the report under *License evidence gaps*. +A component whose only texts are a REUSE ``LICENSES/`` pool ships them, and that meets +the obligation below, but a pool says what each file is under, not what the component is. +Nothing is concluded for it, so its licence is ``NOASSERTION``, and it is listed under +*License evidence gaps* too — as is a component whose grants name nothing the grammar +accepts. + A component that states a licence nowhere at all blocks publication, because packaging the text is the obligation the wheel has to meet. For material checked in here that is any one file whose REUSE info names no licence, names one with no current SPDX reference text (a diff --git a/scripts/sbom/sbom/build.py b/scripts/sbom/sbom/build.py index 5878292b20..bd32722952 100644 --- a/scripts/sbom/sbom/build.py +++ b/scripts/sbom/sbom/build.py @@ -557,12 +557,7 @@ def build( "sbom_in_wheel": sbom_name, # What a sibling wheel's document cites to reference this one. "describes": _describes(spdx, metadata_raw, sidecars["spdx"].name), - "components_without_license_evidence": payload[ - "components_without_license_evidence" - ], - "components_with_unidentified_license": payload[ - "components_with_unidentified_license" - ], + **{key: payload[key] for key in report_module.GAP_LISTS}, "sidecars": { kind: { "filename": path.name, diff --git a/scripts/sbom/sbom/cli.py b/scripts/sbom/sbom/cli.py index 660930ffdb..f42986c98a 100644 --- a/scripts/sbom/sbom/cli.py +++ b/scripts/sbom/sbom/cli.py @@ -19,6 +19,7 @@ from . import wheelfile from . import evidence as evidence_module from . import licensing +from . import report as report_module from . import validate as validate_module @@ -93,12 +94,7 @@ def _cmd_build(args: argparse.Namespace) -> int: print(f"manifest: {manifest_path}") for item in merged["wheels"]: - # Both gap lists. They sit in the same record and mean the same kind of - # thing to a reader; surfacing one and not the other hid the stricter one. - for key in ( - "components_without_license_evidence", - "components_with_unidentified_license", - ): + for key in report_module.GAP_LISTS: for entry in item[key]: print( f"::warning::{item['filename']}: {entry['component']}: " diff --git a/scripts/sbom/sbom/document.py b/scripts/sbom/sbom/document.py index 4ece7e95e3..436a52c652 100644 --- a/scripts/sbom/sbom/document.py +++ b/scripts/sbom/sbom/document.py @@ -263,15 +263,17 @@ def _wheel_conclusion(declared: str, inventory: Inventory) -> str: Every component, not only the ones the build could not name: Apache-2.0 does not cover MIT, Zlib, BSD-3-Clause, Qhull or BSL-1.0, so leaving those out said the wheel carries no code under them. Only a component concluding the - declared licence itself adds nothing. + declared licence itself adds nothing. One component with unresolved terms + leaves the whole wheel's unresolved: NOASSERTION cannot be AND-ed. """ + present = inventory.components_present.values() + if any(item.license_concluded == NOASSERTION for item in present): + return NOASSERTION # Decomposed, so an id two components share appears once and a compound one # does not arrive nested -- but never through an `OR`, which is one choice # and becomes a claim on both halves if it is split. operands: set[str] = set() - for component in inventory.components_present.values(): - if component.license_concluded == NOASSERTION: - continue + for component in present: operands.update(licensing.operands_of(component.license_concluded)) if declared != NOASSERTION: operands.update(licensing.operands_of(declared)) diff --git a/scripts/sbom/sbom/licensing.py b/scripts/sbom/sbom/licensing.py index c5ae6882e8..3ed39df980 100644 --- a/scripts/sbom/sbom/licensing.py +++ b/scripts/sbom/sbom/licensing.py @@ -144,6 +144,16 @@ def grant_texts(evidence) -> list[LicenseEvidence]: return [item for item in evidence if item.kind == "grant"] +def pool_texts(evidence) -> list[LicenseEvidence]: + """A REUSE pool's texts: what individual files are under, not the component.""" + return [item for item in evidence if item.kind == "pool"] + + +def pool_names(paths) -> str: + """A pool's texts by the licence each is named for, as `MIT, Zlib`.""" + return ", ".join(sorted(Path(path).stem for path in paths)) + + def carried_texts(evidence) -> list[LicenseEvidence]: """The grants a component's LicenseRef stands for.""" return [item.text for item in _reading(evidence).carried] @@ -703,6 +713,23 @@ def why_unnamed(evidence) -> str: return "; ".join(reasons) +def why_unresolved(evidence) -> str: + """Why a component whose texts shipped concludes NOASSERTION. + + Without a grant, its only texts are a REUSE pool. With one, its grants + yielded no id: the grammar refused to combine them, or an SPDX reference + text carries no identified id. + """ + if grants := grant_texts(evidence): + paths = ", ".join(sorted(item.path for item in grants)) + return f"its grants ({paths}) yield no SPDX expression" + pooled = pool_names(item.path for item in pool_texts(evidence)) + return ( + f"only REUSE pool texts ship ({pooled}); no grant states which of them " + "are the component's terms" + ) + + def _listed() -> _Listed: if _LISTED is None: raise CorpusError("SPDX license list data has not been loaded") diff --git a/scripts/sbom/sbom/report.py b/scripts/sbom/sbom/report.py index 74174b1daa..4257fae9a6 100644 --- a/scripts/sbom/sbom/report.py +++ b/scripts/sbom/sbom/report.py @@ -6,7 +6,6 @@ from __future__ import annotations import re -from pathlib import Path from . import licensing from .inventory import Inventory @@ -41,6 +40,15 @@ _COMMIT = re.compile(r"^[0-9a-f]{40}$") +# Every list of components whose terms a reader cannot take as identified, with +# its Markdown heading. The report, its Markdown, the manifest and the CI +# warnings all read this, so a list added here cannot be left out of one. +GAP_LISTS = { + "components_without_license_evidence": "No license text found for:", + "components_with_unresolved_license": "No license concluded for:", + "components_with_unidentified_license": "License text carried verbatim:", +} + def _short(version: str) -> str: """Abbreviate a commit; leave anything else whole. @@ -218,12 +226,10 @@ def _carried_reason(evidence) -> str: def report(inventory: Inventory, evidence_doc: dict, wheel_name: str) -> dict: """Machine-readable component-to-license report, including what is missing.""" components = [] - no_evidence = [] - unidentified = [] + gaps: dict[str, list[dict]] = {key: [] for key in GAP_LISTS} for key, component in inventory.components_present.items(): obligations = licensing.obligation_texts(component.evidence) - carried = licensing.carried_texts(component.evidence) components.append( { "component": key, @@ -247,18 +253,28 @@ def report(inventory: Inventory, evidence_doc: dict, wheel_name: str) -> dict: # text. Requiring a grant here advertised "no licence evidence" for a # component the gates deliberately accepted on the pool they packaged. if not obligations: - no_evidence.append( + gaps["components_without_license_evidence"].append( { "component": key, "reason": "no license grant found in the component's own files", } ) - elif carried: + elif component.license_concluded == NOASSERTION: + # Texts that pass the gate can still name nothing + # (`licensing.why_unresolved`). + gaps["components_with_unresolved_license"].append( + { + "component": key, + "reason": licensing.why_unresolved(component.evidence), + "evidence": [item.path for item in obligations], + } + ) + elif carried := licensing.carried_texts(component.evidence): # Not whether anything matched: a text can match and still be # carried (`licensing.carried_texts`), and one beside it can match # and be named. Either way these terms are only readable as the # text this wheel carries. - unidentified.append( + gaps["components_with_unidentified_license"].append( { "component": key, "reason": _carried_reason(component.evidence), @@ -287,8 +303,7 @@ def report(inventory: Inventory, evidence_doc: dict, wheel_name: str) -> dict: {"component": key, "reason": reason} for key, reason in sorted(inventory.absent_components.items()) ], - "components_without_license_evidence": no_evidence, - "components_with_unidentified_license": unidentified, + **gaps, } @@ -335,7 +350,7 @@ def report_markdown(payload: dict) -> str: for record in shown ] if pooled: - names = ", ".join(sorted(Path(record["path"]).stem for record in pooled)) + names = licensing.pool_names(record["path"] for record in pooled) parts.append(f"REUSE pool ({len(pooled)}): {names}") evidence = "
".join(parts) or "—" how = "; ".join( @@ -387,19 +402,14 @@ def report_markdown(payload: dict) -> str: lines.append("None.") lines += ["", "## License evidence gaps", ""] - gaps = payload["components_without_license_evidence"] - unidentified = payload["components_with_unidentified_license"] - if gaps: - lines += ["No license text found for:", ""] - lines += [f"- `{item['component']}` — {item['reason']}" for item in gaps] - lines.append("") - if unidentified: - lines += ["License text carried verbatim:", ""] + listed = [key for key in GAP_LISTS if payload[key]] + for key in listed: + lines += [GAP_LISTS[key], ""] lines += [ - f"- `{item['component']}` — {item['reason']}" for item in unidentified + f"- `{item['component']}` — {item['reason']}" for item in payload[key] ] lines.append("") - if not gaps and not unidentified: + if not listed: lines.append( "None; every redistributed component ships an identified license." if payload["components"] diff --git a/tests/python/sbom/test_licensing.py b/tests/python/sbom/test_licensing.py index bfdf7fbdd5..acf8fb12e9 100644 --- a/tests/python/sbom/test_licensing.py +++ b/tests/python/sbom/test_licensing.py @@ -6,6 +6,7 @@ from __future__ import annotations import math +from dataclasses import replace import pytest import synth @@ -122,8 +123,10 @@ def test_a_real_license_survives_surrounding_packaging_boilerplate(): assert [license_id for license_id, _, _ in licensing.identify(wrapped)] == ["Zlib"] -def _grant(key: str, name: str, text: str) -> licensing.LicenseEvidence: - return licensing.evidence_for(key, "component-file", f"{key}/{name}", text, "grant") +def _grant( + key: str, name: str, text: str, kind: str = "grant" +) -> licensing.LicenseEvidence: + return licensing.evidence_for(key, "component-file", f"{key}/{name}", text, kind) def test_unmatched_grant_becomes_a_license_ref_not_a_guess(): @@ -386,6 +389,30 @@ def test_the_report_says_whether_a_carried_grant_is_named_too(mit_is_newer): ) +def test_grants_concluding_nothing_are_not_blamed_on_a_pool(monkeypatch): + """Grants conclude nothing where the SPDX grammar rejects their ids or a + reference text names none.""" + mit = _grant("demo", "LICENSE", synth.spdx_text("MIT")) + pool = _grant("demo", "LICENSES/MIT.txt", mit.text, kind="pool") + unnamed = replace(mit, origin=licensing.SPDX_REFERENCE, identified=None) + reason = "its grants (demo/LICENSE) yield no SPDX expression" + + assert licensing.expression([unnamed, pool], "demo")[0] == "NOASSERTION" + assert licensing.why_unresolved([unnamed, pool]) == reason + + def rejected(*_args, **_kwargs): + raise licensing.ExpressionError("rejected") + + monkeypatch.setattr(licensing, "combine_expressions", rejected) + + assert licensing.expression([mit, pool], "demo")[0] == "NOASSERTION" + assert licensing.why_unresolved([mit, pool]) == reason + assert licensing.why_unresolved([pool]) == ( + "only REUSE pool texts ship (MIT); no grant states which of them are " + "the component's terms" + ) + + def test_no_grant_at_all_is_noassertion(): assert licensing.expression([], "demo") == ("NOASSERTION", "NOASSERTION") diff --git a/tests/python/sbom/test_wheel_sbom.py b/tests/python/sbom/test_wheel_sbom.py index ec23e2850b..b0e55aa3ec 100644 --- a/tests/python/sbom/test_wheel_sbom.py +++ b/tests/python/sbom/test_wheel_sbom.py @@ -12,6 +12,7 @@ import pytest import synth from sbom import build as build_module +from sbom import report as report_module from sbom import validate as validate_module from sbom import wheelfile @@ -29,17 +30,27 @@ def _embedded(wheel): return json.loads(archive.read(f"{DIST_INFO}/sboms/{WHEEL_NAME}.spdx.json")) -def test_wheel_verifies_against_its_own_sbom(built): - assert validate_module.check(built["wheel"]) == [] +def _wheel_package(spdx: dict) -> dict: + return next( + item for item in spdx["packages"] if item["SPDXID"] == "SPDXRef-Package-wheel" + ) -def test_release_validation_passes(built): - failures = validate_module.check( +def _release_failures(built) -> list[str]: + """What `check` says of the wheel with its build evidence and manifest.""" + return validate_module.check( built["wheel"], build_evidence=built["evidence_path"], manifest=built["manifest_path"], ) - assert failures == [] + + +def test_wheel_verifies_against_its_own_sbom(built): + assert validate_module.check(built["wheel"]) == [] + + +def test_release_validation_passes(built): + assert _release_failures(built) == [] def test_sbom_lands_in_dist_info_sboms_beside_any_other_tools(built): @@ -190,16 +201,78 @@ def test_an_unmatched_grant_beside_a_matched_one_is_carried_and_reported( "named: matched no text in the SPDX reference corpus", "evidence": ["alpha-src/LICENSE-EULA.txt"], } - assert ( - validate_module.check( - built["wheel"], - build_evidence=built["evidence_path"], - manifest=built["manifest_path"], - ) - == [] + assert _release_failures(built) == [] + + +def test_a_component_shipping_only_a_reuse_pool_is_a_reported_gap(workspace): + """The pool passes the gate and concludes nothing, so it cannot fall + between the lists and leave the summary saying every licence is known.""" + source = workspace.build / "_deps" / "alpha-src" + (source / "LICENSES").mkdir() + (source / "LICENSE").rename(source / "LICENSES" / "MIT.txt") + synth.track(workspace.root) + built = synth.collect_and_bind(workspace) + + report = built["licenses"] + alpha = next(item for item in report["components"] if item["component"] == "alpha") + assert alpha["license_concluded"] == "NOASSERTION" + listing = { + key: [item for item in report[key] if item["component"] == "alpha"] + for key in report_module.GAP_LISTS + } + assert [key for key, found in listing.items() if found] == [ + "components_with_unresolved_license" + ] + [entry] = listing["components_with_unresolved_license"] + assert entry["evidence"] == ["alpha-src/LICENSES/MIT.txt"] + assert entry["reason"].startswith("only REUSE pool texts ship (MIT)") + [record] = built["manifest"]["wheels"] + assert record["components_with_unresolved_license"] == [entry] + # The wheel's own conclusion cannot AND in terms nobody concluded. + assert _wheel_package(built["spdx"])["licenseConcluded"] == "NOASSERTION" + assert _release_failures(built) == [] + + +def _report_of(**gaps) -> dict: + """A report payload with one identified component and the given gap lists.""" + row = { + "name": "alpha", + "kind": "source-tree", + "version": "1", + "license_concluded": "MIT", + "how_it_reaches_the_wheel": [], + "evidence": [], + } + return { + "wheel": WHEEL_NAME, + "generated_from_commit": None, + "spdx_license_list_version": None, + "components": [row], + "externally_supplied_runtime": [], + "consumer_requirements": [], + "exclusions": [], + **{key: gaps.get(key, []) for key in report_module.GAP_LISTS}, + } + + +def test_an_identified_set_reports_no_gaps(): + rendered = report_module.report_markdown(_report_of()) + + assert rendered.endswith( + "## License evidence gaps\n\n" + "None; every redistributed component ships an identified license.\n" ) +@pytest.mark.parametrize("key", list(report_module.GAP_LISTS)) +def test_any_gap_list_withholds_the_all_identified_summary(key): + gap = {"component": "alpha", "reason": "why"} + rendered = report_module.report_markdown(_report_of(**{key: [gap]})) + + assert f"{report_module.GAP_LISTS[key]}\n\n- `alpha` — why\n" in rendered + assert "None; every redistributed component" not in rendered + + def _rewrite_alpha(built, expression: str) -> list[str]: """Rewrite alpha's expressions in the embedded document; what check says.""" member = f"{DIST_INFO}/sboms/{WHEEL_NAME}.spdx.json" @@ -535,10 +608,7 @@ def test_a_wheel_shipping_no_source_still_names_its_licensor(workspace, license_ ).decode() members = set(archive.namelist()) - wheel_package = next( - item for item in spdx["packages"] if item["SPDXID"] == "SPDXRef-Package-wheel" - ) - assert wheel_package["copyrightText"] != "NOASSERTION" + assert _wheel_package(spdx)["copyrightText"] != "NOASSERTION" assert "Copyright" in notices assert "Example Org" in notices # The SPDX spelling of a supplier is not prose. From f521a32dc430807988e2524bd08b41e5d1d95352 Mon Sep 17 00:00:00 2001 From: Andrei Aristarkhov Date: Tue, 20 Oct 2026 04:03:29 -0700 Subject: [PATCH 085/101] docs(sbom): REUSE terms are read only for checked-in files; a sidecar is no grant sbom.rst said terms stated in source headers stand in for a licence file for every component. Only checked-in third-party files are read that way; a fetched tree's headers, sidecars and REUSE.toml decide nothing, so one with no licence file at its root, no LICENSES/ pool and no README licence section blocks publication. That was not true either: a REUSE `.license` sidecar at a fetched tree's root was taken as its grant, so a two-line SPDX tag shipped as its terms. A file is now a sidecar only when the file it annotates sits beside it, and a sidecar is never a grant: at a tree's root, an archive's root, beside a toolkit library, or as a README. Signed-off-by: Andrei Aristarkhov --- docs/source/references/sbom.rst | 50 ++++++++++++++++------------- scripts/sbom/sbom/discovery.py | 17 +++++++--- scripts/sbom/sbom/evidence.py | 3 +- scripts/sbom/sbom/licensing.py | 39 +++++++++++++++++----- tests/python/sbom/test_discovery.py | 25 +++++++++++++++ tests/python/sbom/test_gates.py | 12 +++++++ tests/python/sbom/test_licensing.py | 49 ++++++++++++++++++++++++++++ 7 files changed, 160 insertions(+), 35 deletions(-) diff --git a/docs/source/references/sbom.rst b/docs/source/references/sbom.rst index a71b9cda85..fd7d04e540 100644 --- a/docs/source/references/sbom.rst +++ b/docs/source/references/sbom.rst @@ -209,13 +209,15 @@ maintained by hand. REUSE metadata is not such a list: it is how this repository the terms of each file it holds — an in-file ``SPDX-License-Identifier``, a ``.license`` sidecar or a ``REUSE.toml`` annotation, one declaration — and the pre-commit hook lints it for source files. The collector reads it for every file here that REUSE covers, whatever its -type, and for every source the build compiled or included: a file identified as a third +type, and for every source here the build compiled or included: a file identified as a third party's (see below) is held to the per-file gate even where the hook does not lint it, and a -file naming no holder at all counts as this project's own. For each fetched component the -collector finds the license and notice files the component itself ships — ``LICENSE``, -``COPYING``, ``BSD-LICENSE``, a ``LICENSES/`` pool, or, where upstream ships no file at all, -the license section of its README — and matches each text against a corpus of SPDX reference -texts. +file naming no holder at all counts as this project's own. A fetched component's licence is +not read from its source headers, ``.license`` sidecars or a ``REUSE.toml``: for each +fetched tree the collector finds the license and notice files the component itself ships — +``LICENSE``, ``COPYING``, ``BSD-LICENSE``, a ``LICENSES/`` pool, or, where none of those at +its root is a grant, the license section of its README — and matches each text against a +corpus of SPDX reference texts. An archive in the repository is read only for the license +files at its root. Those reference texts are a dependency like any other. :code-file:`deps/third_party/CMakeLists.txt` fetches @@ -322,21 +324,24 @@ Nothing is concluded for it, so its licence is ``NOASSERTION``, and it is listed *License evidence gaps* too — as is a component whose grants name nothing the grammar accepts. -A component that states a licence nowhere at all blocks publication, because packaging the -text is the obligation the wheel has to meet. For material checked in here that is any one -file whose REUSE info names no licence, names one with no current SPDX reference text (a -``LicenseRef-`` among them, even beside one that has a text), states an expression that does -not parse, or names an id that is neither SPDX-listed nor a ``LicenseRef-`` or -``DocumentRef-``; an exception needs only to be listed as an exception. The build names each -such file and why, and another file of the same holder does not lend it its terms. The +A component with no licence text the collector reads blocks publication, because packaging +the text is the obligation the wheel has to meet. For a fetched component that is no licence +file at its root and, for a fetched tree, none in a ``LICENSES/`` pool and no licence +section in its README; a licence file in a subdirectory covers only that part, and terms +only its REUSE info states do not count. For material checked in here it is any one file +whose REUSE info names no licence, names one with no current SPDX reference text (a +``LicenseRef-`` among them, even beside one that has a text), states an expression that +does not parse, or names an id that is neither SPDX-listed nor a ``LicenseRef-`` or +``DocumentRef-``; an exception needs only to be listed as an exception. The build names +each such file and why, and another file of the same holder does not lend it its terms. The remedy is to record upstream's terms as an ``SPDX-License-Identifier`` in the header, a ``.license`` sidecar or a ``REUSE.toml`` annotation. A tag that is not one — quoted in a string, say, or prose that mentions the tag — goes between ``REUSE-IgnoreStart`` and ``REUSE-IgnoreEnd``, or is set aside by a ``.license`` sidecar or a ``REUSE.toml`` -annotation with ``precedence = "override"``. Where REUSE declares an identifier, the -reference text for it ships, labelled in the document as a substitution rather than as the -component's own words — it conveys the terms and names no holder, so the file's copyright -notice travels with it. +annotation with ``precedence = "override"``. Where a checked-in third-party file's REUSE +info declares an identifier, the reference text for it ships, labelled in the document as a +substitution rather than as the component's own words — it conveys the terms and names no +holder, so the file's copyright notice travels with it. A shared library that none of those indexes explains did not come from this build, so the last thing asked is the build machine itself: resolve the library's SONAME, and take its @@ -435,11 +440,12 @@ inventory with its identity, its license and the route it took to get there. These do need attention: -- **A component that states its license nowhere stops the build.** The fix is to obtain the - terms, not to record an assumption. For a file checked in here, obtain the terms, then - record them as REUSE licence info naming current SPDX-listed identifiers; a stray tag - that does not parse or names no such id is set aside instead. Both remedies are under - `How licenses are determined`_. +- **A component with no license text the collector reads stops the build.** The fix is to + obtain the terms, not to record an assumption; a fetched component's REUSE info, source + headers included, is not read for them. For a file checked in here, obtain the terms, + then record them as REUSE licence info naming current SPDX-listed identifiers; a stray + tag that does not parse or names no such id is set aside instead. Both remedies are + under `How licenses are determined`_. - **A new member that the build cannot explain stops the build.** That means something reached the wheel by a route the collector cannot see; teach it that route rather than excluding the file. diff --git a/scripts/sbom/sbom/discovery.py b/scripts/sbom/sbom/discovery.py index e53e28a1ac..5c381ecb7c 100644 --- a/scripts/sbom/sbom/discovery.py +++ b/scripts/sbom/sbom/discovery.py @@ -2170,7 +2170,9 @@ def _license_near(library: Path) -> Path | None: levels = 3 if root is None else len(current.relative_to(root).parts) + 1 for _ in range(levels): for entry in sorted(current.iterdir()) if current.is_dir() else []: - if entry.is_file() and licensing.classify(entry.name): + if entry.is_file() and licensing.classify( + entry.name, licensing.files_in(current) + ): return entry if current.parent == current: break @@ -2290,6 +2292,11 @@ def add_archive(self, path: Path, display: str) -> None: "size": size, "member_count": len(members), } + at_root = frozenset( + Path(member.path).name + for member, _ in members + if Path(member.path).parent.as_posix() in {".", ""} + ) for member, data in members: # `data` is the whole member only at the archive root; elsewhere it # is the head kept for exactly this, so ask only for the build-id. @@ -2300,9 +2307,11 @@ def add_archive(self, path: Path, display: str) -> None: self.by_hash.setdefault(member.sha256, member) self.all_by_hash.setdefault(member.sha256, []).append(member) self.by_name.setdefault(Path(member.path).name, []).append(member) - self._maybe_license(display, member, data) + self._maybe_license(display, member, data, at_root) - def _maybe_license(self, display: str, member: Member, data: bytes) -> None: + def _maybe_license( + self, display: str, member: Member, data: bytes, at_root: frozenset[str] + ) -> None: name = Path(member.path).name if Path(member.path).parent.as_posix() not in {".", ""}: return # only the archive root states the bundle's own terms @@ -2312,7 +2321,7 @@ def _maybe_license(self, display: str, member: Member, data: bytes) -> None: text = data.decode("utf-8", "replace").strip().splitlines() self.archives[display]["version_text"] = text[0].strip() if text else "" return - if not licensing.classify(name): + if not licensing.classify(name, at_root.__contains__): return self.archives[display].setdefault("license_members", []).append((member, data)) diff --git a/scripts/sbom/sbom/evidence.py b/scripts/sbom/sbom/evidence.py index b6800a478c..66c8e0f6de 100644 --- a/scripts/sbom/sbom/evidence.py +++ b/scripts/sbom/sbom/evidence.py @@ -133,7 +133,8 @@ def _archive_components( "archive-member", f"{display}!{member.path}", data.decode("utf-8", "replace"), - licensing.classify(Path(member.path).name) or "grant", + # Sidecars were set aside when the member was kept. + licensing.classify(Path(member.path).name, lambda _: False) or "grant", ) for member, data in record.get("license_members", []) ] diff --git a/scripts/sbom/sbom/licensing.py b/scripts/sbom/sbom/licensing.py index 3ed39df980..13fb3d97b3 100644 --- a/scripts/sbom/sbom/licensing.py +++ b/scripts/sbom/sbom/licensing.py @@ -14,6 +14,7 @@ import json import math import re +from collections.abc import Callable from functools import lru_cache from dataclasses import dataclass, replace from typing import TYPE_CHECKING @@ -527,8 +528,20 @@ def evidence_for( ) -def classify(name: str) -> str | None: - if Path(name).suffix.lower() in _SKIP_SUFFIXES: +def files_in(directory: Path) -> Callable[[str], bool]: + """Whether `directory` holds a file of a given name: `is_sidecar`'s `beside`.""" + return lambda name: (directory / name).is_file() + + +def is_sidecar(name: str, beside: Callable[[str], bool]) -> bool: + """A REUSE `.license` sidecar: the annotation of a file `beside` finds next + to it. Named so with nothing to annotate, it is a licence file.""" + annotated = name.removesuffix(".license") + return annotated not in {name, ""} and beside(annotated) + + +def classify(name: str, beside: Callable[[str], bool]) -> str | None: + if Path(name).suffix.lower() in _SKIP_SUFFIXES or is_sidecar(name, beside): return None if _GRANT_NAMES.search(name): return "grant" @@ -545,7 +558,7 @@ def discover_in_tree( candidates: list[tuple[Path, str]] = [] for entry in sorted(root.iterdir()) if root.is_dir() else []: - if entry.is_file() and (kind := classify(entry.name)): + if entry.is_file() and (kind := classify(entry.name, files_in(root))): candidates.append((entry, kind)) elif entry.is_dir() and entry.name.upper() in {"LICENSES", "LICENSE"}: candidates.extend( @@ -562,7 +575,11 @@ def discover_in_tree( path.is_file() and path.parent != root and path.parent.name.upper() not in {"LICENSES", "LICENSE"} - and (classify(path.name) or _NOTICE_NAMES.match(path.name)) + and ( + classify(path.name, files_in(path.parent)) + or _NOTICE_NAMES.match(path.name) + or is_sidecar(path.name, files_in(path.parent)) + ) ): candidates.append((path, "nested")) @@ -573,9 +590,11 @@ def discover_in_tree( display = f"{display_prefix}/{path.relative_to(root).as_posix()}" item = evidence_for(component, "component-file", display, text, kind) if kind == "nested": - annotated = path.name.removesuffix(".license") - sidecar = path.with_name(annotated) if annotated else path - covered = sidecar if sidecar != path and sidecar.is_file() else path.parent + covered = ( + path.with_name(path.name.removesuffix(".license")) + if is_sidecar(path.name, files_in(path.parent)) + else path.parent + ) scope = f"{display_prefix}/{covered.relative_to(root).as_posix()}" item = replace(item, scope=scope) found.append(item) @@ -591,7 +610,11 @@ def _readme_fallback( ) -> list[LicenseEvidence]: """Some upstreams state the grant in a README section and ship no license file.""" for entry in sorted(root.iterdir()) if root.is_dir() else []: - if not entry.is_file() or not _README_NAMES.match(entry.name): + if ( + not entry.is_file() + or not _README_NAMES.match(entry.name) + or is_sidecar(entry.name, files_in(root)) + ): continue text = _read(entry) if text is None: diff --git a/tests/python/sbom/test_discovery.py b/tests/python/sbom/test_discovery.py index eff0dc58f6..7031b9c040 100644 --- a/tests/python/sbom/test_discovery.py +++ b/tests/python/sbom/test_discovery.py @@ -8,6 +8,7 @@ import json import os import subprocess +import zipfile import zlib from pathlib import Path @@ -485,6 +486,30 @@ def test_archive_version_and_license_come_from_the_archive(workspace): assert [member.path for member, _ in record["license_members"]] == ["LICENSE.txt"] +def test_an_archive_root_sidecar_is_not_a_license_member(tmp_path): + archive = tmp_path / "sdk.zip" + with zipfile.ZipFile(archive, "w") as handle: + handle.writestr("LICENSE.txt", synth.spdx_text("MIT")) + handle.writestr("libdemo.so", b"\0") + handle.writestr("libdemo.so.license", "SPDX-License-Identifier: MIT\n") + handle.writestr("stb.license", synth.spdx_text("MIT")) + index = discovery.ArchiveIndex() + + index.add_archive(archive, "sdk.zip") + + assert [ + member.path for member, _ in index.archives["sdk.zip"]["license_members"] + ] == ["LICENSE.txt", "stb.license"] + + +def test_a_toolkit_licence_beside_a_library_is_not_a_sidecar(tmp_path): + synth.write(tmp_path / "lib/libx.so.1", "\0") + synth.write(tmp_path / "lib/libx.so.1.license", "SPDX-License-Identifier: MIT\n") + synth.write(tmp_path / "EULA.txt", "terms\n") + + assert discovery._license_near(tmp_path / "lib/libx.so.1") == tmp_path / "EULA.txt" + + def test_a_build_tree_without_a_file_api_reply_says_so(workspace): """The query lives in CMakeLists, so a missing reply means the tree was configured by a CMake too old to honour it — say that, don't guess.""" diff --git a/tests/python/sbom/test_gates.py b/tests/python/sbom/test_gates.py index 73d2cfca5f..2f349c9e57 100644 --- a/tests/python/sbom/test_gates.py +++ b/tests/python/sbom/test_gates.py @@ -47,6 +47,18 @@ def test_a_component_with_no_license_text_blocks_the_build(workspace): _rebuild(workspace, synth.wheel_payload(workspace)) +def test_a_root_sidecar_is_not_a_fetched_component_s_license_text(workspace): + alpha = workspace.build / "_deps/alpha-src" + (alpha / "LICENSE").unlink() + (alpha / "README.md").write_text("# alpha\n", encoding="utf-8") + (alpha / "README.md.license").write_text( + "SPDX-License-Identifier: MIT\n", encoding="utf-8" + ) + + with pytest.raises(build_module.BuildError, match=r"no license text.*alpha"): + _rebuild(workspace, synth.wheel_payload(workspace)) + + def test_an_empty_license_file_does_not_count_as_evidence(workspace): # delta reaches the wheel through a static archive, so emptying its license # leaves the wheel's contents untouched and isolates the evidence gate. diff --git a/tests/python/sbom/test_licensing.py b/tests/python/sbom/test_licensing.py index acf8fb12e9..9f5b10c6e8 100644 --- a/tests/python/sbom/test_licensing.py +++ b/tests/python/sbom/test_licensing.py @@ -602,6 +602,55 @@ def test_an_index_of_licenses_is_not_a_grant(tmp_path): assert licensing.discover_in_tree("catalogue", tmp_path, "catalogue-src") == [] +@pytest.mark.parametrize( + ("name", "beside", "expected"), + [ + ("README.md.license", {"README.md"}, None), + ("LICENSE.license", {"LICENSE"}, None), + ("stb.license", set(), "grant"), + (".license", {""}, "grant"), + ], +) +def test_a_license_suffix_is_a_sidecar_only_beside_what_it_annotates( + name, beside, expected +): + assert licensing.classify(name, beside.__contains__) == expected + + +def test_a_root_sidecar_is_not_a_fetched_tree_s_grant(tmp_path): + """Nor is it a README whose licence section the fallback could read.""" + (tmp_path / "README.md").write_text("# Thing\n\nprose\n", encoding="utf-8") + (tmp_path / "README.md.license").write_text( + "SPDX-License-Identifier: MIT\n\n## License\nMIT\n", encoding="utf-8" + ) + + assert licensing.discover_in_tree("thing", tmp_path, "thing-src") == [] + + +def test_root_license_named_files_are_kept_unless_they_annotate_one(tmp_path): + synth.write(tmp_path / "LICENSE", synth.spdx_text("MIT")) + synth.write(tmp_path / "stb.license", synth.spdx_text("MIT")) + synth.write(tmp_path / ".license", synth.spdx_text("MIT")) + synth.write(tmp_path / "mit", "data\n") + synth.write(tmp_path / "mit.license", "SPDX-License-Identifier: MIT\n") + + evidence = licensing.discover_in_tree("thing", tmp_path, "thing-src") + + assert [(item.path, item.kind) for item in evidence] == [ + ("thing-src/.license", "grant"), + ("thing-src/LICENSE", "grant"), + ("thing-src/stb.license", "grant"), + ] + + +def test_an_orphan_license_file_alone_is_a_fetched_tree_s_grant(tmp_path): + synth.write(tmp_path / "stb.license", synth.spdx_text("MIT")) + + (item,) = licensing.discover_in_tree("thing", tmp_path, "thing-src") + + assert (item.kind, item.identified) == ("grant", "MIT") + + def test_a_project_title_is_not_a_license_section(tmp_path): """`# License List Data` heads a README; it does not grant anything.""" (tmp_path / "README.md").write_text( From fcb9303540c95a2c56e86d34bef407a428b7924e Mon Sep 17 00:00:00 2001 From: Andrei Aristarkhov Date: Tue, 20 Oct 2026 04:10:11 -0700 Subject: [PATCH 086/101] refactor(sbom): drop the unused document-reference machinery No document carries externalDocumentRefs: a sibling wheel is named as a package, not bound to its document's digest. Yet the CLI still built wheels in dependency order, recorded each document's `describes` digest in the manifest and overwrote sibling facts from it, all to support a reference nothing emits and nothing reads. Remove in_dependency_order, _describes, the manifest's `describes` key and the never-filled external-document list. siblings_of already supplies every sibling field the document reads, and the validator still checks any externalDocumentRefs a consumer meets. Signed-off-by: Andrei Aristarkhov --- scripts/sbom/sbom/build.py | 68 ++--------------------------------- scripts/sbom/sbom/cli.py | 10 +----- scripts/sbom/sbom/document.py | 2 -- 3 files changed, 3 insertions(+), 77 deletions(-) diff --git a/scripts/sbom/sbom/build.py b/scripts/sbom/sbom/build.py index bd32722952..9e92a6b1e9 100644 --- a/scripts/sbom/sbom/build.py +++ b/scripts/sbom/sbom/build.py @@ -21,7 +21,6 @@ from . import report as report_module from . import validate as validate_module from . import wheelfile -from packaging.requirements import Requirement from packaging.utils import canonicalize_name from . import discovery as discovery_module @@ -200,9 +199,8 @@ def siblings_of(wheels: list[Path], repo_root: Path) -> dict[str, dict]: Read before any of them is described, so a requirement naming one is never reported as an install-time unknown. Two wheels can require each other -- a - renamed project and its transition wheel do -- and only one of them can - carry a digest-bound reference to the other's document; both can state what - the metadata says, because that needs no document. + renamed project and its transition wheel do -- and both can state what the + metadata says, because that needs no document. """ known: dict[str, dict] = {} for wheel in wheels: @@ -218,66 +216,6 @@ def siblings_of(wheels: list[Path], repo_root: Path) -> dict[str, dict]: return known -def in_dependency_order(wheels: list[Path]) -> list[Path]: - """Wheels of one run, each after any sibling it requires. - - A document that references another cites its digest, so the referenced one - has to exist first. Order is derived from the requirements rather than from - the filenames, which only happen to sort the right way. - """ - metadata = {} - for wheel in wheels: - _, parsed = wheelfile.read_metadata(wheel, wheelfile.dist_info_of(wheel)) - metadata[wheel] = ( - canonicalize_name(parsed["name"]), - { - canonicalize_name(Requirement(item).name) - for item in parsed.get("requires_dist") or [] - }, - ) - - ordered: list[Path] = [] - placed: set[str] = set() - remaining = list(wheels) - while remaining: - ready = [ - wheel - for wheel in remaining - if not ( - {name for name, _ in metadata.values()} - & metadata[wheel][1] - placed - {metadata[wheel][0]} - ) - ] - # A cycle between two wheels of one run cannot be ordered; keep the - # caller's order rather than refusing to describe either. - batch = ready or remaining - for wheel in batch: - ordered.append(wheel) - placed.add(metadata[wheel][0]) - remaining = [item for item in remaining if item not in batch] - return ordered - - -def _describes(spdx: dict, metadata_raw: bytes, sbom_filename: str) -> dict: - """What another wheel's document needs to cite this one, SPDX 2.3 style.""" - metadata = wheelfile.parse_metadata(metadata_raw) - body = json.dumps(spdx, indent=2, sort_keys=True).encode("utf-8") + b"\n" - wheel_package = next( - item - for item in spdx["packages"] - if item["SPDXID"] == document_module.WHEEL_PACKAGE_ID - ) - return { - "name": metadata["name"], - "version": metadata.get("version") or "NOASSERTION", - "namespace": spdx["documentNamespace"], - "sha1": hashlib.sha1(body).hexdigest(), # noqa: S324 - "filename": sbom_filename, - "license_declared": wheel_package["licenseDeclared"], - "supplier": wheel_package["supplier"], - } - - def build( repo_root: Path, build_dir: Path, @@ -555,8 +493,6 @@ def build( "sha256": sha256_file(wheel_path), "size": wheel_path.stat().st_size, "sbom_in_wheel": sbom_name, - # What a sibling wheel's document cites to reference this one. - "describes": _describes(spdx, metadata_raw, sidecars["spdx"].name), **{key: payload[key] for key in report_module.GAP_LISTS}, "sidecars": { kind: { diff --git a/scripts/sbom/sbom/cli.py b/scripts/sbom/sbom/cli.py index f42986c98a..e7dbbf17ab 100644 --- a/scripts/sbom/sbom/cli.py +++ b/scripts/sbom/sbom/cli.py @@ -11,8 +11,6 @@ import zipfile from pathlib import Path -from packaging.utils import canonicalize_name - from . import TOOL_NAME, TOOL_VERSION from . import build as build_module from . import discovery @@ -61,11 +59,8 @@ def _cmd_build(args: argparse.Namespace) -> int: manifests: list[dict] = [] wheels = sorted(Path(item) for item in args.wheel) - # What every wheel of this run says about itself, before any is described. - # A document reference needs the referenced document to exist; these facts - # do not, so a requirement on a sibling is never an install-time unknown. siblings = build_module.siblings_of(wheels, repo_root) - for wheel in build_module.in_dependency_order(wheels): + for wheel in wheels: manifest = build_module.build( repo_root, build_dir, @@ -76,9 +71,6 @@ def _cmd_build(args: argparse.Namespace) -> int: siblings=siblings, ) manifests.append(manifest) - described = manifest["wheels"][0]["describes"] - siblings[canonicalize_name(described["name"])] = described - siblings[canonicalize_name(described["name"])]["wheel"] = wheel.name print(f"packaged evidence into {wheel.name}") # Through the same merge as a release: inheriting the first wheel's manifest diff --git a/scripts/sbom/sbom/document.py b/scripts/sbom/sbom/document.py index 436a52c652..89217f488e 100644 --- a/scripts/sbom/sbom/document.py +++ b/scripts/sbom/sbom/document.py @@ -296,7 +296,6 @@ def build_document( """ files: list[dict] = [] file_ids: dict[str, str] = {} - external_documents: list[dict] = [] sha1_digests: list[str] = [] # What the files themselves state, for the package-level roll-up below. @@ -602,5 +601,4 @@ def build_document( "files": files, "relationships": relationships, "hasExtractedLicensingInfos": extracted, - **({"externalDocumentRefs": external_documents} if external_documents else {}), } From b9c9e722885aae816f4b44c7c8ebd261827ca9ee Mon Sep 17 00:00:00 2001 From: Andrei Aristarkhov Date: Tue, 20 Oct 2026 04:11:56 -0700 Subject: [PATCH 087/101] docs(sbom): say what each collector option is for build --wheel is rewritten in place and --out-dir receives the sidecars and manifest.json, but neither said so in --help; nor did check --wheel, check-set --manifest/--evidence-dir or merge-manifests' arguments. Every option now has a one-line description. Signed-off-by: Andrei Aristarkhov --- scripts/sbom/sbom/cli.py | 27 ++++++++++++++++++++------- 1 file changed, 20 insertions(+), 7 deletions(-) diff --git a/scripts/sbom/sbom/cli.py b/scripts/sbom/sbom/cli.py index e7dbbf17ab..b311ccceda 100644 --- a/scripts/sbom/sbom/cli.py +++ b/scripts/sbom/sbom/cli.py @@ -140,8 +140,15 @@ def main(argv: list[str] | None = None) -> int: help="discover what a build put in its wheels, then package the evidence into them", ) build.add_argument("--build-dir", required=True, help="configured CMake build tree") - build.add_argument("--wheel", required=True, action="append") - build.add_argument("--out-dir", required=True) + build.add_argument( + "--wheel", + required=True, + action="append", + help="wheel to describe; rewritten in place, repeatable", + ) + build.add_argument( + "--out-dir", required=True, help="receives the sidecars and manifest.json" + ) build.add_argument( "--license-data", help=( @@ -155,7 +162,9 @@ def main(argv: list[str] | None = None) -> int: "check", help="check a wheel against the SBOM it carries, plus any evidence you supply", ) - check.add_argument("--wheel", required=True, action="append") + check.add_argument( + "--wheel", required=True, action="append", help="wheel to check; repeatable" + ) check.add_argument( "--build-evidence", help="also check that the build accounts for every member" ) @@ -170,20 +179,24 @@ def main(argv: list[str] | None = None) -> int: check_set = subparsers.add_parser( "check-set", help="run check over every wheel a merged manifest advertises" ) - check_set.add_argument("--manifest", required=True) + check_set.add_argument( + "--manifest", required=True, help="merged manifest of the release" + ) check_set.add_argument( "--wheel-dir", required=True, help="the release's wheels; one the manifest does not list fails the set", ) - check_set.add_argument("--evidence-dir", required=True) + check_set.add_argument( + "--evidence-dir", required=True, help="where the release's sidecars live" + ) check_set.set_defaults(handler=_cmd_check_set) merge = subparsers.add_parser( "merge-manifests", help="fold per-variant manifests into one" ) - merge.add_argument("--output", required=True) - merge.add_argument("manifest", nargs="+") + merge.add_argument("--output", required=True, help="merged manifest to write") + merge.add_argument("manifest", nargs="+", help="per-variant manifests") merge.set_defaults(handler=_cmd_merge_manifests) args = parser.parse_args(argv) From efdef39aa90321630966c87431df756a1c04aa7d Mon Sep 17 00:00:00 2001 From: Andrei Aristarkhov Date: Tue, 20 Oct 2026 04:12:57 -0700 Subject: [PATCH 088/101] docs(sbom): say these releases carry no PEP 740 attestations sbom.rst called PEP 740 attestations "the provenance mechanism", which read as if these releases supplied them. publish-wheel uploads with a plain twine upload and no id-token, so they do not. Say it is the ecosystem's mechanism and that these releases are not published with attestations. Signed-off-by: Andrei Aristarkhov --- docs/source/references/sbom.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/source/references/sbom.rst b/docs/source/references/sbom.rst index fd7d04e540..6d8068ea67 100644 --- a/docs/source/references/sbom.rst +++ b/docs/source/references/sbom.rst @@ -47,7 +47,8 @@ Integrity and provenance are the packaging ecosystem's job, not this document's: `PEP 740 `_ attestations Signed attestations binding a publisher identity to an artifact's hash, served - through PyPI's Integrity API. That is the provenance mechanism. + through PyPI's Integrity API. That is the ecosystem's mechanism for binding + provenance to a distribution; these releases are not published with attestations. The SBOM *describes* what those files are — where each came from, under what license. Its per-file digests are there so the description can be shown to belong to the wheel in From 8cf1410c7af13f3d4183030c3d31577e4bfe0109 Mon Sep 17 00:00:00 2001 From: Andrei Aristarkhov Date: Tue, 20 Oct 2026 04:15:46 -0700 Subject: [PATCH 089/101] refactor(sbom): delete code nothing uses _INCLUDE_FLAG and FIRST_PARTY had no users, read_notice was called only by tests that read_notices covers, and siblings_of filled name and wheel keys that no document reads. Signed-off-by: Andrei Aristarkhov --- scripts/sbom/sbom/build.py | 2 -- scripts/sbom/sbom/discovery.py | 1 - scripts/sbom/sbom/inventory.py | 2 -- scripts/sbom/sbom/licensing.py | 6 ------ tests/python/sbom/test_licensing.py | 11 +++++------ 5 files changed, 5 insertions(+), 17 deletions(-) diff --git a/scripts/sbom/sbom/build.py b/scripts/sbom/sbom/build.py index 9e92a6b1e9..30bfbb9c9a 100644 --- a/scripts/sbom/sbom/build.py +++ b/scripts/sbom/sbom/build.py @@ -207,11 +207,9 @@ def siblings_of(wheels: list[Path], repo_root: Path) -> dict[str, dict]: _, parsed = wheelfile.read_metadata(wheel, wheelfile.dist_info_of(wheel)) facts = _project_facts(repo_root, parsed) known[canonicalize_name(parsed["name"])] = { - "name": parsed["name"], "version": parsed.get("version") or NOASSERTION, "supplier": facts["supplier"], "license_declared": facts["license"], - "wheel": wheel.name, } return known diff --git a/scripts/sbom/sbom/discovery.py b/scripts/sbom/sbom/discovery.py index 5c381ecb7c..ced8301b8c 100644 --- a/scripts/sbom/sbom/discovery.py +++ b/scripts/sbom/sbom/discovery.py @@ -84,7 +84,6 @@ _VERSION_IN_NAME = re.compile(r"(\d+\.\d+[\w.+-]*?)(?=-|\.tar|\.tgz|\.zip|$)") # The ref ExternalProject's clone script checks out; `clone --no-checkout` is not one. _GITCLONE_TAG = re.compile(r'(?[^"]+)"') -_INCLUDE_FLAG = re.compile(r"-(?:I|isystem)\s*(\S+)") _DEPS_SOURCE = re.compile(r"_deps/(?P[A-Za-z0-9_.+-]+)-src(?:/|$)") diff --git a/scripts/sbom/sbom/inventory.py b/scripts/sbom/sbom/inventory.py index ef6aa44600..88e1da42d2 100644 --- a/scripts/sbom/sbom/inventory.py +++ b/scripts/sbom/sbom/inventory.py @@ -37,8 +37,6 @@ if TYPE_CHECKING: # pragma: no cover from .evidence import Discovery -FIRST_PARTY = "__first_party__" - # How a component that reached no file got into the build, by kind. Checked-in # sources were never fetched, and the collector's own vocabulary says so. _ARRIVED = { diff --git a/scripts/sbom/sbom/licensing.py b/scripts/sbom/sbom/licensing.py index 13fb3d97b3..e2de110432 100644 --- a/scripts/sbom/sbom/licensing.py +++ b/scripts/sbom/sbom/licensing.py @@ -1341,12 +1341,6 @@ def _names_a_holder(notice: str) -> bool: return bool(re.search(r"[A-Za-z]{2}", rest)) -def read_notice(text: str) -> str | None: - """The first notice a licence text states, where only one is wanted.""" - notices = read_notices(text) - return notices[0] if notices else None - - def read_copyright(text: str) -> str | None: """The notice a file states about itself, beside its REUSE tag.""" match = _FILE_COPYRIGHT.search(text) diff --git a/tests/python/sbom/test_licensing.py b/tests/python/sbom/test_licensing.py index 9f5b10c6e8..8d4baff6b7 100644 --- a/tests/python/sbom/test_licensing.py +++ b/tests/python/sbom/test_licensing.py @@ -701,12 +701,11 @@ def test_folding_does_not_merge_different_holders(): def test_a_template_is_not_a_notice(): """A reference text names nobody; recording its placeholder would.""" - assert licensing.read_notice("Copyright [yyyy] [name of copyright owner]") is None - assert licensing.read_notice("Copyright (c) ") is None - assert ( - licensing.read_notice("MIT\n\nCopyright (c) 2016 Wenzel Jakob\n\nPermission") - == "Copyright (c) 2016 Wenzel Jakob" - ) + assert licensing.read_notices("Copyright [yyyy] [name of copyright owner]") == [] + assert licensing.read_notices("Copyright (c) ") == [] + assert licensing.read_notices( + "MIT\n\nCopyright (c) 2016 Wenzel Jakob\n\nPermission" + ) == ["Copyright (c) 2016 Wenzel Jakob"] def test_a_debian_copyright_field_names_its_holder_either_way(license_data): From 8736249e6b4cdfd0d90391577d854d1db44b9464 Mon Sep 17 00:00:00 2001 From: Andrei Aristarkhov Date: Tue, 20 Oct 2026 04:18:21 -0700 Subject: [PATCH 090/101] docs(sbom): state the rules comments protect, not the bugs that taught them Comments in the collector recounted earlier failures (a set dated before its last wheel, a pinned sibling, an entry naming only the last components, a pool text credited to an SDK). Restate each as the invariant it protects, per the repo's comment rules. Signed-off-by: Andrei Aristarkhov --- scripts/sbom/sbom/build.py | 14 +++++++------- scripts/sbom/sbom/cli.py | 5 ++--- scripts/sbom/sbom/discovery.py | 4 ++-- scripts/sbom/sbom/document.py | 14 ++++++-------- scripts/sbom/sbom/inventory.py | 4 ++-- scripts/sbom/sbom/report.py | 5 ++--- 6 files changed, 21 insertions(+), 25 deletions(-) diff --git a/scripts/sbom/sbom/build.py b/scripts/sbom/sbom/build.py index 30bfbb9c9a..feb6fa07be 100644 --- a/scripts/sbom/sbom/build.py +++ b/scripts/sbom/sbom/build.py @@ -333,8 +333,8 @@ def build( # The distribution's own declared licence is an obligation like any other. # Apache-2.0 asks that recipients get a copy, and the gate above covers only - # what the build redistributes -- so a wheel could declare a licence whose - # text it does not carry, and the transition wheel did. + # what the build redistributes -- so a wheel can declare a licence whose + # text it does not carry, as a metadata-only wheel does. declared = _project_facts(repo_root, wheel_metadata)["license"] shipped = { report_module.declared_license_path(wheel.dist_info, item.name) @@ -393,15 +393,15 @@ def build( projected_wheel = replace(wheel, entries=tuple(projected)) for name in [*additions, *replacements]: # Only the third-party texts: the notices file and the document below - # are this collector's own output, and reading them back found the first - # `- Copyright:` bullet of the summary it had just generated. + # are this collector's own output, and reading them back would take a + # `- Copyright:` bullet of its own summary for a notice. packaged_text = name in additions and report_module.is_third_party_license( wheel.dist_info, name ) body = additions.get(name, b"").decode("utf-8", "replace") - # A licence text can state its own terms and carry a notice, and these - # were the only members never read for either -- the collector packaged - # them, so nothing scanned them the way it scans what it is describing. + # A licence text can state its own terms and carry a notice. The + # collector packages these itself, so it reads them here, the way it + # reads every member it describes. inventory.attributions[name] = Attribution( path=name, origin="metadata", diff --git a/scripts/sbom/sbom/cli.py b/scripts/sbom/sbom/cli.py index b311ccceda..4b18f285bd 100644 --- a/scripts/sbom/sbom/cli.py +++ b/scripts/sbom/sbom/cli.py @@ -73,9 +73,8 @@ def _cmd_build(args: argparse.Namespace) -> int: manifests.append(manifest) print(f"packaged evidence into {wheel.name}") - # Through the same merge as a release: inheriting the first wheel's manifest - # dated the set before the last wheel it binds existed, and skipped the - # duplicate-filename check. + # The merge a release uses: the set is stamped after its last wheel, and one + # filename cannot stand for two wheels. merged = build_module.merge_records( [item for manifest in manifests for item in manifest["wheels"]] ) diff --git a/scripts/sbom/sbom/discovery.py b/scripts/sbom/sbom/discovery.py index ced8301b8c..9f39052292 100644 --- a/scripts/sbom/sbom/discovery.py +++ b/scripts/sbom/sbom/discovery.py @@ -212,8 +212,8 @@ def notice_from(evidence) -> str: """The copyright a component's own licence text states, where it states one. Permissive licences generally require the notice to travel with the terms, - and for MIT and the BSDs it is a line of the same file -- shipped in the - wheel but, until now, named nowhere a reader could act on. + and for MIT and the BSDs it is a line of the same file; this states it + where a reader can act on it. """ # Grants only, though a pool counts as licence evidence everywhere else: a # pool is a directory of reference texts, and the notice in one is the diff --git a/scripts/sbom/sbom/document.py b/scripts/sbom/sbom/document.py index 89217f488e..6206274b53 100644 --- a/scripts/sbom/sbom/document.py +++ b/scripts/sbom/sbom/document.py @@ -130,8 +130,8 @@ def _extracted_entries(components: list[Component]) -> list[dict]: f"{ref} was minted for two components with different terms; " "a LicenseRef identifies one text" ) - # Accumulated, not replaced: three components sharing one text left - # the entry naming the last two and citing only the last one's files. + # Accumulated, not replaced: the entry names every component sharing + # the text and cites all of their files. covers = set(entries.get(ref, {}).get("_for", ())) | {component.name} cited = sorted( set(entries.get(ref, {}).get("_from", ())) @@ -518,12 +518,10 @@ def build_document( ) else: # Built by this same run: not an install-time unknown, so the facts - # its own metadata states are recorded here. Deliberately not a - # digest-bound reference to that wheel's document -- a `py3-none-any` - # wheel is built by every matrix entry, and pinning one entry's - # sibling made the universal wheel differ by who built it. It also - # kept the two documents asymmetric, since a requirement cycle means - # only one of them could ever carry the reference. + # its own metadata states are recorded here. Not a digest-bound + # reference to that wheel's document: a `py3-none-any` wheel is built + # by every matrix entry and must not differ by which one built it, + # and in a requirement cycle only one document could carry it. element = package_id comment = ( f"{declared} Built by this same build; its own document " diff --git a/scripts/sbom/sbom/inventory.py b/scripts/sbom/sbom/inventory.py index 88e1da42d2..8e15ac2587 100644 --- a/scripts/sbom/sbom/inventory.py +++ b/scripts/sbom/sbom/inventory.py @@ -696,8 +696,8 @@ def _resolve_by_path( # This repository first. A member matching a file this project tracks is # this project's copy; a dependency whose tree happens to end in the same - # two path segments is a coincidence -- our own BSL-1.0 pool text was - # being credited to an SDK that ships one under the same relative path. + # two path segments is a coincidence, as when an SDK ships a BSL-1.0 + # text under the same relative path as our pool's. # A third-party file checked in here still carries its component, because # the repository branch looks its holder up. repo_file = self.repo_files.path_suffix_match(wheel_path) diff --git a/scripts/sbom/sbom/report.py b/scripts/sbom/sbom/report.py index 4257fae9a6..88f3c1fdc6 100644 --- a/scripts/sbom/sbom/report.py +++ b/scripts/sbom/sbom/report.py @@ -102,9 +102,8 @@ def packaged_license_path(dist_info: str, component_key: str, evidence) -> str: """ folder = _SAFE_NAME.sub("_", component_key) relative = evidence.path.rsplit("!", 1)[-1].split("/", 1)[-1] - # Keep the separators: flattening them re-created the collision this whole - # scheme exists to avoid, and the loser was dropped while the document went - # on naming the path it was dropped from. + # Keep the separators: flattening them re-creates the collision this scheme + # exists to avoid, and the document would name a text that is not there. safe = "/".join(_SAFE_NAME.sub("_", part) for part in relative.split("/")) return f"{licenses_root(dist_info)}third-party/{folder}/{safe}" From 638604365f6051a83374c119df0172356ff06f9c Mon Sep 17 00:00:00 2001 From: Andrei Aristarkhov Date: Tue, 20 Oct 2026 14:00:54 -0700 Subject: [PATCH 091/101] fix(sbom): name a host package's vendor by the archive that published it A system package's purl took its namespace from the build machine's os-release, so CUDA installed from NVIDIA's apt repository was published as pkg:deb/ubuntu/cuda-cudart-dev-*: a vendor claim that vulnerability matching acts on. The namespace is now the apt Origin of the sources listing the installed version, or, where no archive lists that version, of those listing its other versions; it must be a single Origin, compared ignoring case. A package whose vendor cannot be named that way (a local .deb, archives that disagree or state no Origin, an Origin that is a URL or host name) is pkg:generic/@, since pkg:deb requires its vendor. Build evidence records apt_origin and apt_origin_from. The owner is read only from dpkg-query -S's answer line, so translated diversion lines are never taken for a package and a diverted path names none; apt and dpkg-query -W are asked with the architecture-qualified name dpkg reports. apt output is parsed by layout, not by its translated headings, and tolerates phased-update notes. Signed-off-by: Andrei Aristarkhov --- docs/source/references/sbom.rst | 21 +- scripts/sbom/sbom/discovery.py | 136 +++++++-- tests/python/sbom/conftest.py | 5 +- tests/python/sbom/test_static_archives.py | 318 +++++++++++++++++++++- 4 files changed, 445 insertions(+), 35 deletions(-) diff --git a/docs/source/references/sbom.rst b/docs/source/references/sbom.rst index 6d8068ea67..8ded029bcb 100644 --- a/docs/source/references/sbom.rst +++ b/docs/source/references/sbom.rst @@ -349,12 +349,21 @@ last thing asked is the build machine itself: resolve the library's SONAME, and license from the supplying package's copyright file, or from a license file beside the library for toolchains that install outside the package manager. Resolving the SONAME is what licenses the claim — a library the machine cannot find is reported as unexplained, -not attributed to something nobody located. No repair tool's directory name is consulted, -so a library the build produced is attributed to the build wherever it ends up. Its -copyright notice is read only from the ``Copyright:`` fields of a machine-readable Debian -copyright file, leaving out stanzas that cover only the Debian packaging; any other -text — an EULA, a free-form copyright file — quotes the notices of what it bundles, so -none is asserted, and the packaged text still carries them all. +not attributed to something nobody located. The package's ``pkg:deb`` purl is namespaced +by the archive that publishes the installed version — the ``Origin`` apt records for it, +``ubuntu`` for Ubuntu's, ``nvidia`` for NVIDIA's CUDA repository. Where no archive lists +the installed version — superseded, typically, but equally a local rebuild — the archives +listing its other versions decide, and the build evidence says so. ``pkg:deb`` requires +that vendor, so where none can be named — no archive lists the package, as for a local +``.deb``; the deciding archives disagree, ignoring case, or name no ``Origin``; or the +``Origin`` is a URL or a host name, which a purl namespace cannot be — the package is +``pkg:generic/@`` instead. The build machine's own distribution is not +consulted. No repair tool's directory name is consulted, so a library the build produced +is attributed to the build wherever it ends up. Its copyright notice is read only from +the ``Copyright:`` fields of a machine-readable Debian copyright file, leaving out +stanzas that cover only the Debian packaging; any other text — an EULA, a free-form +copyright file — quotes the notices of what it bundles, so none is asserted, and the +packaged text still carries them all. Static code leaves no ``DT_NEEDED`` entry behind, so the link line is where it is read — every fragment CMake records for it, ``-Wl,`` and ``-Xlinker`` words included, with the diff --git a/scripts/sbom/sbom/discovery.py b/scripts/sbom/sbom/discovery.py index 9f39052292..b983d6b35b 100644 --- a/scripts/sbom/sbom/discovery.py +++ b/scripts/sbom/sbom/discovery.py @@ -23,7 +23,6 @@ import json import logging import os -import platform import re import shlex import subprocess @@ -2088,36 +2087,110 @@ def archive_members(path: Path) -> int | None: def _identify_host_package(library: Path, origin: dict) -> dict: - """Name the distro package that owns a resolved host library.""" - owned = _ask("dpkg-query", "-S", str(library)) - if ":" not in owned: + """Name the package dpkg says owns a resolved host library.""" + # dpkg names a multiarch package with its architecture, `libc6:i386`, and + # dpkg-query -W and apt answer for one architecture only when asked so. + qualified = _dpkg_owner(library) + if not qualified: return origin - package = owned.split(":", 1)[0].strip() + package = qualified.partition(":")[0] origin["package"] = package - origin["supplier"] = _package_supplier(package) - origin["version"] = _package_field(package, "Version") - origin["distro"] = _os_release_id() + origin["supplier"] = _package_supplier(qualified) + origin["version"] = _package_field(qualified, "Version") + origin["apt_origin"], origin["apt_origin_from"] = _apt_origin(qualified) copyright_file = Path("/usr/share/doc") / package / "copyright" if copyright_file.is_file(): origin["copyright"] = str(copyright_file) return origin +# A package name as dpkg prints it, with its architecture where it has one. +_DPKG_PACKAGE = re.compile(r"^[a-z0-9][a-z0-9+.-]+(?::[a-z0-9-]+)?$") + + +def _dpkg_owner(library: Path) -> str | None: + """The first package `dpkg-query -S` names as owning `library`. + + Its last line answers, `[, ...]: `; diversion lines + come before it, translated. Where `library` is diverted, the file there may + not be the listed package's, so none is named. + """ + lines = _ask("dpkg-query", "-S", str(library)).splitlines() + suffix = f": {library}" + if not lines or any(line.endswith(suffix) for line in lines[:-1]): + return None + if not lines[-1].endswith(suffix): + return None + owner = lines[-1].removesuffix(suffix).split(",", 1)[0].strip() + return owner if _DPKG_PACKAGE.match(owner) else None + + def _package_field(package: str, field: str) -> str | None: - """One field of a distro package's metadata, as dpkg records it.""" + """One field of a package's metadata, as dpkg records it.""" return _ask("dpkg-query", "-W", f"-f=${{{field}}}", package).strip() or None -def _os_release_id() -> str | None: - """The distribution this package manager belongs to, for the purl type.""" - try: - return platform.freedesktop_os_release().get("ID") or None - except OSError: - return None +# `apt-cache policy` lines. Its headings are translated; these are not. With no +# package named: a source (`%4i `) and, directly under it, its +# Release fields. In a version table: a version (`***` when installed, perhaps +# with a note such as `(phased 20%)`) and, indented further, a source listing it. +_APT_SOURCE = re.compile(r"^(?P *)-?\d+ (?P\S.*?)\s*$") +_APT_RELEASE = re.compile(r"^ {5}release (?P.*)$") +_APT_RELEASE_FIELD = re.compile(r",(?=[a-z]=)") +_APT_VERSION = re.compile(r"^ (?P\*\*\*| ) \S+ -?\d+(?: .*)?$") + + +@functools.lru_cache(maxsize=1) +def _apt_releases() -> dict[str, dict[str, str]]: + """Each source apt knows, with the fields of its Release file (`o` is Origin).""" + releases: dict[str, dict[str, str]] = {} + source = None + for line in _ask("apt-cache", "policy").splitlines(): + listed = _APT_SOURCE.match(line) + if listed and len(listed["indent"]) <= 3: + source = listed["source"] + releases[source] = {} + elif source and (release := _APT_RELEASE.match(line)): + releases[source] = dict( + item.partition("=")[::2] + for item in _APT_RELEASE_FIELD.split(release["fields"]) + ) + source = None + else: + source = None + return releases + + +@functools.lru_cache(maxsize=None) +def _apt_origin(package: str) -> tuple[str | None, str | None]: + """`(Origin, "installed version")` where the archives listing the installed + version agree on one, ignoring case; else `(Origin, "other versions")` where + none lists it and those listing other versions agree; else `(None, None)`. + """ + releases = _apt_releases() + # Installed or not: lowercased Origin ("" for none) -> first spelling. + origins: dict[bool, dict[str, str]] = {True: {}, False: {}} + installed = False + for line in _ask("apt-cache", "policy", package).splitlines(): + if version := _APT_VERSION.match(line): + installed = version["installed"] == "***" + elif (listed := _APT_SOURCE.match(line)) and len(listed["indent"]) >= 5: + fields = releases.get(listed["source"], {}) + # dpkg's status file, release `a=now`, says what is installed, not + # where it came from. + if fields.get("a") != "now": + origin = fields.get("o", "") + origins[installed].setdefault(origin.lower(), origin) + decided_by_installed = bool(origins[True]) + found = origins[decided_by_installed] + if len(found) == 1 and "" not in found: + decided = "installed version" if decided_by_installed else "other versions" + return next(iter(found.values())), decided + return None, None def _package_supplier(package: str) -> str | None: - """Who ships a distro package, in the form SPDX wants. + """Who ships a package dpkg can name, in the form SPDX wants. The package name is not a supplier: `libbsd0` names the thing, not whoever provided it. dpkg records a maintainer, which is the answer. @@ -2193,6 +2266,25 @@ def _system_notice(evidence: list[licensing.LicenseEvidence]) -> str: return "NOASSERTION" +def _system_purl(soname: str, origin: dict) -> str: + """`pkg:deb//...` for a package an archive's Origin names, else generic. + + The deb type requires its namespace, the vendor, lowercased. ECMA-427 5.6.3 + keeps `/` out of a namespace segment and a URL host out of a namespace, so + an Origin that is a URL or a host name names no vendor either. + """ + package = origin.get("package") + vendor = (origin.get("apt_origin") or "").lower() if package else "" + if "/" in vendor or ("." in vendor and not re.search(r"\s", vendor)): + vendor = "" + return PackageURL( + type="deb" if vendor else "generic", + namespace=vendor or None, + name=package or soname, + version=origin.get("version"), + ).to_string() + + def system_component( soname: str, origin: dict, route: str | None = None, key: str | None = None ) -> Component: @@ -2232,17 +2324,7 @@ def system_component( supplier=origin.get("supplier") or "NOASSERTION", copyright_text=_system_notice(evidence), homepage="NOASSERTION", - # A distro package the machine can name resolves; pkg:generic does not. - purl=( - PackageURL( - type="deb", - namespace=origin.get("distro"), - name=origin["package"], - version=origin.get("version"), - ).to_string() - if origin.get("package") - else PackageURL(type="generic", name=soname).to_string() - ), + purl=_system_purl(soname, origin), version=origin.get("version") or "NOASSERTION", download_location="NOASSERTION", source_info=" ".join(details), diff --git a/tests/python/sbom/conftest.py b/tests/python/sbom/conftest.py index 64d9c01bec..3245672683 100644 --- a/tests/python/sbom/conftest.py +++ b/tests/python/sbom/conftest.py @@ -63,8 +63,11 @@ def _license_data_loaded(license_data): @pytest.fixture(autouse=True) def _fresh_checkout_answers(): - """Each test builds its own checkouts; a cached answer is a previous test's.""" + """Each test builds its own checkouts and fakes its own host; a cached + answer is a previous test's.""" discovery._is_own_checkout.cache_clear() + discovery._apt_releases.cache_clear() + discovery._apt_origin.cache_clear() @pytest.fixture(scope="session") diff --git a/tests/python/sbom/test_static_archives.py b/tests/python/sbom/test_static_archives.py index 4f3b349bf4..72e7c420a4 100644 --- a/tests/python/sbom/test_static_archives.py +++ b/tests/python/sbom/test_static_archives.py @@ -5,8 +5,11 @@ from __future__ import annotations +from pathlib import Path + import pytest import synth +from packageurl import PackageURL from sbom import build as build_module from sbom import discovery from sbom import document as document_module @@ -549,7 +552,8 @@ def test_a_packaged_archive_is_inventoried_through_its_package( copyright_file, supplier="Organization: Example Distro (devel@example.com)", version="1.0-1", - distro="ubuntu", + apt_origin="Ubuntu", + apt_origin_from="installed version", ) _link(workspace, str(archive)) @@ -563,6 +567,8 @@ def test_a_packaged_archive_is_inventoried_through_its_package( evidence = collected["build-evidence"] record = evidence["static_inputs"][str(archive)] assert record["package"] == "libsquash-dev" + assert record["apt_origin"] == "Ubuntu" + assert record["apt_origin_from"] == "installed version" assert record["linked_by"] == ["ext"] linked = next( item for item in evidence["link_artifacts"] if item["target"] == "ext" @@ -570,6 +576,316 @@ def test_a_packaged_archive_is_inventoried_through_its_package( assert linked["static_inputs"] == [str(archive)] +# `apt-cache policy` as apt prints it. NVIDIA's CUDA repository is a flat one, +# `deb / /`, listed with an empty distribution. +UBUNTU_SOURCE = "http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 Packages" +NVIDIA_SOURCE = ( + "https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64" + " Packages" +) +SECURITY_SOURCE = "http://security.ubuntu.com/ubuntu jammy-security/main amd64 Packages" +FLAT_SOURCE = "https://repo.example.com/apt ./ Packages" +# A repository with no Release file states no Origin. +BARE_SOURCE = "file:/srv/debs ./ Packages" +# libnvidia-container's Origin is a URL. +URL_ORIGIN = "https://nvidia.github.io/libnvidia-container/stable" +URL_SOURCE = f"{URL_ORIGIN}/deb/amd64 Packages" +# A mirror spelling Ubuntu's Origin in lower case. +MIRROR_SOURCE = "http://mirror.example.org/ubuntu jammy-updates/main amd64 Packages" +HOST_SOURCE = "https://packages.example.com/apt stable/main amd64 Packages" +APT_SOURCES = f"""Package files: + 100 /var/lib/dpkg/status + release a=now + 600 {NVIDIA_SOURCE} + release o=NVIDIA,l=NVIDIA CUDA,c= + origin developer.download.nvidia.com + 500 {UBUNTU_SOURCE} + release v=22.04,o=Ubuntu,a=jammy-updates,n=jammy,l=Ubuntu,c=main,b=amd64 + origin archive.ubuntu.com + 500 {FLAT_SOURCE} + release o=Example,c= + origin repo.example.com + 500 {SECURITY_SOURCE} + release v=22.04,o=Ubuntu,a=jammy-security,n=jammy,l=Ubuntu,c=main,b=amd64 + origin security.ubuntu.com + 500 {BARE_SOURCE} + 500 {URL_SOURCE} + release o={URL_ORIGIN},l=NVIDIA CORPORATION ,c= + origin nvidia.github.io + 500 {MIRROR_SOURCE} + release o=ubuntu,a=jammy-updates,c=main,b=amd64 + origin mirror.example.org + 500 {HOST_SOURCE} + release o=packages.example.com,a=stable,c=main,b=amd64 + origin packages.example.com +Pinned packages: +""" + + +def _version_table(*versions: tuple) -> str: + """`(version, installed, sources[, note])` per version, as apt lists them.""" + lines = ["libsquash-dev:", " Installed: 1.0-1", " Version table:"] + for version, installed, sources, *note in versions: + lines.append( + " ".join([f" {'***' if installed else ' '} {version} 500", *note]) + ) + lines += [f" 500 {source}" for source in sources] + return "\n".join(lines) + "\n" + + +STATUS = "/var/lib/dpkg/status" +LIBRARY = "/usr/lib/libsquash.a" + + +def _host_package( + monkeypatch, + policy, + owned=f"libsquash-dev:amd64: {LIBRARY}\n", + qualified="libsquash-dev:amd64", + sources=None, +): + """What `_identify_host_package` makes of dpkg answering `owned`, and what it + asked. dpkg-query -W and apt answer only for `qualified`.""" + answers = { + ("dpkg-query", "-S", LIBRARY): owned, + ("dpkg-query", "-W", "-f=${Version}", qualified): "1.0-1", + ("apt-cache", "policy"): APT_SOURCES if sources is None else sources, + ("apt-cache", "policy", qualified): policy, + } + asked = [] + + def ask(*command, env=None): + asked.append(command) + return answers.get(command, "") + + monkeypatch.setattr(discovery, "_ask", ask) + return discovery._identify_host_package(Path(LIBRARY), {}), asked + + +@pytest.mark.parametrize( + ("policy", "namespace", "decided"), + [ + pytest.param( + _version_table( + ("1.0-1", True, [UBUNTU_SOURCE, STATUS]), + ("0.9-1", False, [FLAT_SOURCE]), + ), + "ubuntu", + "installed version", + id="installed-version-decides-not-another", + ), + pytest.param( + _version_table(("1.0-1", True, [NVIDIA_SOURCE, STATUS])), + "nvidia", + "installed version", + id="nvidia", + ), + pytest.param( + _version_table(("1.0-1", True, [FLAT_SOURCE])), + "example", + "installed version", + id="flat", + ), + pytest.param( + _version_table(("1.0-1", True, [UBUNTU_SOURCE, FLAT_SOURCE, STATUS])), + None, + None, + id="disagreeing", + ), + pytest.param( + _version_table( + ("1.0-1", True, [BARE_SOURCE, STATUS]), + ("0.9-1", False, [UBUNTU_SOURCE]), + ), + None, + None, + id="no-origin", + ), + # As for libc6 in an ubuntu:22.04 image. + pytest.param( + _version_table( + ("1.1-1", False, [UBUNTU_SOURCE, SECURITY_SOURCE]), + ("1.0-1", True, [STATUS]), + ("0.9-1", False, [UBUNTU_SOURCE]), + ), + "ubuntu", + "other versions", + id="installed-version-unlisted-other-versions-decide", + ), + pytest.param( + _version_table( + ("1.1-1", False, [UBUNTU_SOURCE]), + ("1.0-1", True, [STATUS]), + ("0.9-1", False, [FLAT_SOURCE]), + ), + None, + None, + id="installed-version-unlisted-others-disagree", + ), + pytest.param( + _version_table(("1.0-1", True, [STATUS])), None, None, id="local-deb" + ), + pytest.param("", None, None, id="apt-knows-nothing"), + # A phased update carries a note after its priority, translated. + pytest.param( + _version_table( + ("1.0-1", True, [UBUNTU_SOURCE, STATUS], "(phased 20%)"), + ("0.9-1", False, [FLAT_SOURCE]), + ), + "ubuntu", + "installed version", + id="phased-installed", + ), + pytest.param( + _version_table( + ("1.0-1", True, [NVIDIA_SOURCE, STATUS]), + ("0.9-1", False, [UBUNTU_SOURCE], "(phased 20%)"), + ), + "nvidia", + "installed version", + id="phased-after-installed", + ), + pytest.param( + _version_table( + ("1.0-1", True, [NVIDIA_SOURCE, STATUS]), + ("0.9-1", False, [UBUNTU_SOURCE], "(gestaffelt 20%)"), + ), + "nvidia", + "installed version", + id="phased-german", + ), + pytest.param( + _version_table(("1.0-1", True, [UBUNTU_SOURCE, MIRROR_SOURCE, STATUS])), + "ubuntu", + "installed version", + id="origins-agree-ignoring-case", + ), + pytest.param( + _version_table(("1.0-1", True, [URL_SOURCE, STATUS])), + None, + "installed version", + id="url-origin-is-no-namespace", + ), + pytest.param( + _version_table(("1.0-1", True, [HOST_SOURCE, STATUS])), + None, + "installed version", + id="host-origin-is-no-namespace", + ), + ], +) +def test_a_packages_namespace_is_the_archive_that_publishes_it( + monkeypatch, policy, namespace, decided +): + origin, _ = _host_package(monkeypatch, policy) + + purl = discovery.system_component("libsquash.a", origin).purl + + # pkg:deb requires its vendor namespace; without one, nothing is claimed. + expected = f"pkg:deb/{namespace}/" if namespace else "pkg:generic/" + assert purl == f"{expected}libsquash-dev@1.0-1" + parsed = PackageURL.from_string(purl) + assert (parsed.type, parsed.namespace, parsed.name, parsed.version) == ( + "deb" if namespace else "generic", + namespace, + "libsquash-dev", + "1.0-1", + ) + assert origin["apt_origin_from"] == decided + + +@pytest.mark.parametrize( + ("owned", "qualified"), + [ + pytest.param( + f"libsquash-dev, libsquash-extra:amd64: {LIBRARY}\n", + "libsquash-dev", + id="several-owners-the-first-as-printed", + ), + pytest.param( + f"libsquash-dev:i386: {LIBRARY}\n", + "libsquash-dev:i386", + id="foreign-architecture", + ), + ], +) +def test_dpkg_and_apt_are_asked_about_the_owner_dpkg_names( + monkeypatch, owned, qualified +): + origin, asked = _host_package( + monkeypatch, + _version_table(("1.0-1", True, [UBUNTU_SOURCE])), + owned=owned, + qualified=qualified, + ) + + assert origin["package"] == "libsquash-dev" + assert origin["version"] == "1.0-1" + assert origin["apt_origin"] == "Ubuntu" + assert { + ("dpkg-query", "-W", "-f=${Version}", qualified), + ("dpkg-query", "-W", "-f=${Maintainer}", qualified), + ("apt-cache", "policy", qualified), + } <= set(asked) + + +@pytest.mark.parametrize( + "owned", + [ + pytest.param( + f"diversion by squash-gl from: {LIBRARY}\n" + f"diversion by squash-gl to: {LIBRARY}.distrib\n", + id="diverted-unowned", + ), + pytest.param( + f"diversion by squash-gl from: {LIBRARY}\n" + f"diversion by squash-gl to: {LIBRARY}.distrib\n" + f"libsquash-dev:amd64: {LIBRARY}\n", + id="diverted-the-file-may-not-be-the-owners", + ), + pytest.param( + f"diversion by squash-gl to: {LIBRARY}\n", + id="only-a-diversion-names-the-path", + ), + ], +) +def test_a_diversion_line_names_no_owner(monkeypatch, owned): + origin, _ = _host_package(monkeypatch, "", owned=owned) + + assert origin.get("package") is None + purl = discovery.system_component("libsquash.a", origin).purl + assert purl == "pkg:generic/libsquash.a" + + +def test_apt_sources_are_read_whatever_the_language(monkeypatch): + """Headings are translated; a pinned package may be named `release`.""" + sources = ( + APT_SOURCES.replace("Package files:", "Fichiers du paquet :").replace( + "Pinned packages:", "Paquets épinglés :" + ) + + " release -> 2.0-1 avec la priorité 990\n" + + " 123 -> 1.0 avec la priorité 1001\n" + ) + _host_package(monkeypatch, "", sources=sources) + + releases = discovery._apt_releases() + + assert set(releases) == { + STATUS, + NVIDIA_SOURCE, + UBUNTU_SOURCE, + FLAT_SOURCE, + SECURITY_SOURCE, + BARE_SOURCE, + URL_SOURCE, + MIRROR_SOURCE, + HOST_SOURCE, + } + assert releases[BARE_SOURCE] == {} + assert releases[NVIDIA_SOURCE]["o"] == "NVIDIA" + assert releases[URL_SOURCE]["o"] == URL_ORIGIN + + # A machine-readable copyright file whose licence paragraph quotes a notice. DEP5 = ( "Format: https://www.debian.org/doc/packaging-manuals/copyright-format/1.0/\n" From 0e905feb3e26b9e3c246ff68758f5079a6469a3a Mon Sep 17 00:00:00 2001 From: Andrei Aristarkhov Date: Tue, 20 Oct 2026 15:30:42 -0700 Subject: [PATCH 092/101] feat(sbom): give each file an SPDXID that survives the next release A file's SPDXID was minted from its position in the sorted wheel listing, so one added file renumbered every later file and every relationship that named one, and two releases' documents could not be diffed. The ID is now the member's path, reduced to the characters an SPDX identifier allows, plus the first 8 hex digits of a sha256 of that path. The versioned -.dist-info/ and .data/ directories read as dist-info/ and data/, so third-party licence texts and metadata keep their IDs across a version bump; the digest separates paths that sanitise alike and, being taken over a form no member name can spell for those two directories, a real top-level data/ tree. Two members that would still share an ID stop the build, naming both. Signed-off-by: Andrei Aristarkhov --- docs/source/references/sbom.rst | 6 ++ scripts/sbom/sbom/document.py | 35 ++++++-- tests/python/sbom/test_wheel_sbom.py | 126 +++++++++++++++++++++++++++ 3 files changed, 162 insertions(+), 5 deletions(-) diff --git a/docs/source/references/sbom.rst b/docs/source/references/sbom.rst index 8ded029bcb..54c37da05d 100644 --- a/docs/source/references/sbom.rst +++ b/docs/source/references/sbom.rst @@ -86,6 +86,12 @@ the sidecars beside it. ``check-set`` runs the whole thing over every wheel in a To read the inventory directly, unzip ``.dist-info/sboms/*.spdx.json`` and feed it to any SPDX 2.3 consumer. CI validates each published document with ``spdx-tools`` before release. +A file's ``SPDXID`` is its path in the distribution, reduced to the characters an SPDX +identifier allows, plus a short digest of the unreduced path, with the versioned +``.dist-info`` and ``.data`` directories counted as fixed. A file keeps its +identifier across releases unless that path changes, so two releases' documents can be +diffed. + Evidence published beside a wheel --------------------------------- diff --git a/scripts/sbom/sbom/document.py b/scripts/sbom/sbom/document.py index 6206274b53..4007ef891a 100644 --- a/scripts/sbom/sbom/document.py +++ b/scripts/sbom/sbom/document.py @@ -39,6 +39,22 @@ def component_package_id(key: str) -> str: return _spdx_id("Package", key) +def file_id(path: str, dist_info: str) -> str: + """The member's id: its path, with the versioned `.dist-info/` and `.data/` + read as `dist-info/` and `data/`, and a digest of that path, which keeps + apart paths that sanitise alike (`a_b.py`, `a-b.py`). + """ + stem = dist_info.removesuffix(".dist-info") + hashed = path + for directory, token in ((dist_info, "dist-info"), (f"{stem}.data", "data")): + if path.startswith(f"{directory}/"): + path = token + path.removeprefix(directory) + # No member name holds a NUL, so a real top-level `data/` hashes apart. + hashed = "\0" + path + break + return _spdx_id("File", f"{path}-{licensing.sha256_text(hashed)[:8]}") + + def verification_code(sha1_digests: list[str]) -> str: """SPDX package verification code: SHA1 over the sorted file SHA1s.""" return hashlib.sha1("".join(sorted(sha1_digests)).encode("ascii")).hexdigest() # noqa: S324 @@ -288,7 +304,8 @@ def build_document( excluded_files: list[str], siblings: dict[str, dict] | None = None, ) -> dict: - """Assemble the document. Callers validate it; this only renders. + """Assemble the document. Callers validate it; this renders, and refuses + two members that would share an SPDXID. `siblings` are the other distributions of this same run, by canonical name. A requirement naming one is not an install-time unknown: the build made it, @@ -304,16 +321,24 @@ def build_document( # file attributed to a component carries that component's notice, which # belongs to that component's package and not to this one. stated_copyright: set[str] = set() - for index, entry in enumerate(sorted(wheel.entries, key=lambda item: item.name)): + paths_by_id: dict[str, str] = {} + for entry in sorted(wheel.entries, key=lambda item: item.name): if entry.name in excluded_files: continue - file_id = _spdx_id("File", f"{index}-{entry.name.rsplit('/', 1)[-1]}") - file_ids[entry.name] = file_id + spdx_id = file_id(entry.name, wheel.dist_info) + # Paths that sanitise alike and share a 32-bit digest prefix. + if spdx_id in paths_by_id: + raise ValueError( + f"{wheel.path.name}: {paths_by_id[spdx_id]} and {entry.name} " + f"would both be {spdx_id}" + ) + paths_by_id[spdx_id] = entry.name + file_ids[entry.name] = spdx_id sha1_digests.append(entry.sha1) attribution = inventory.attributions.get(entry.name) record = { - "SPDXID": file_id, + "SPDXID": spdx_id, "fileName": f"./{entry.name}", "checksums": [ {"algorithm": "SHA1", "checksumValue": entry.sha1}, diff --git a/tests/python/sbom/test_wheel_sbom.py b/tests/python/sbom/test_wheel_sbom.py index b0e55aa3ec..627dec7144 100644 --- a/tests/python/sbom/test_wheel_sbom.py +++ b/tests/python/sbom/test_wheel_sbom.py @@ -8,10 +8,14 @@ import json import re import zipfile +from pathlib import Path import pytest import synth from sbom import build as build_module +from sbom import document as document_module +from sbom import inventory as inventory_module +from sbom import licensing from sbom import report as report_module from sbom import validate as validate_module from sbom import wheelfile @@ -615,3 +619,125 @@ def test_a_wheel_shipping_no_source_still_names_its_licensor(workspace, license_ assert "distributed by Organization:" not in notices # The only text it ships names no holder, which is why the above matters. assert "thin-0.4.0.dist-info/licenses/LICENSES/Apache-2.0.txt" in members + + +_IDSTRING = re.compile(r"SPDXRef-[A-Za-z0-9.-]+") + + +def _file_ids_of(names: list[str], version: str = "0.4.0") -> dict[str, str]: + """Each file's id in a document over a wheel holding only these members.""" + entries = tuple( + wheelfile.Entry(name=name, size=0, sha256="0" * 64, sha1="0" * 40) + for name in names + ) + spdx = document_module.build_document( + { + "supplier": "Organization: Example Org", + "homepage": "https://example.invalid", + "license": "Apache-2.0", + "document_namespace": "https://example.invalid/spdx", + }, + {"license_list_version": "3.27"}, + wheelfile.WheelInfo( + path=Path(f"isaaccapture-{version}-py3-none-any.whl"), + distribution="isaaccapture", + version=version, + dist_info=f"isaaccapture-{version}.dist-info", + entries=entries, + ), + inventory_module.Inventory( + attributions={}, + components_present={}, + component_roles={}, + external_runtime={}, + requires_dist=[], + ), + [], + ) + return { + item["fileName"].removeprefix("./"): item["SPDXID"] for item in spdx["files"] + } + + +def test_a_file_id_is_its_sanitised_path_and_a_digest_of_it(built): + spdx = _embedded(built["wheel"]) + + for item in spdx["files"]: + assert item["SPDXID"] == document_module.file_id( + item["fileName"].removeprefix("./"), DIST_INFO + ) + # Pinned: a change to what is hashed renames every file in every document. + assert ( + document_module.file_id( + "isaaccapture/cloudxr/oob_teleop_progress.py", DIST_INFO + ) + == "SPDXRef-File-isaaccapture-cloudxr-oob-teleop-progress.py-f80fee5e" + ) + assert ( + document_module.file_id(f"{DIST_INFO}/METADATA", DIST_INFO) + == "SPDXRef-File-dist-info-METADATA-ad68becb" + ) + + +def test_adding_a_file_renumbers_no_other_file(): + before = _file_ids_of(["a/one.py", "b/three.py", "c/four.so"]) + after = _file_ids_of(["a/one.py", "a/two.py", "b/three.py", "c/four.so"]) + + assert {name: after[name] for name in before} == before + + +@pytest.mark.parametrize(("old", "new"), [("1.7+local", "1.8"), ("1!1.7", "1!1.8")]) +def test_a_version_bump_renames_no_file(old, new): + def members(version): + return [ + "isaaccapture/cloudxr/oob_teleop_progress.py", + f"isaaccapture-{version}.dist-info/METADATA", + f"isaaccapture-{version}.dist-info/licenses/third-party/fmt/LICENSE", + f"isaaccapture-{version}.data/scripts/isaaccapture-tool", + ] + + before = _file_ids_of(members(old), version=old) + after = _file_ids_of(members(new), version=new) + + assert list(before.values()) == list(after.values()) + + +@pytest.mark.parametrize( + "path", + [ + f"sub/{DIST_INFO}/METADATA", + f"{DIST_INFO}X/f", + f"{DIST_INFO.removesuffix('.dist-info')}.dataX/f", + ], +) +def test_only_the_versioned_directories_are_read_as_fixed(path): + assert document_module.file_id(path, DIST_INFO).startswith( + f"SPDXRef-File-{licensing.spdx_safe(path)}-" + ) + + +def test_paths_that_sanitise_alike_get_distinct_ids(): + ids = _file_ids_of(["pkg/a_b.py", "pkg/a-b.py", "pkg/a b.py", "pkg/a__b.py"]) + + assert len(set(ids.values())) == len(ids) + + +def test_a_path_with_no_safe_character_still_gets_a_valid_id(): + assert _IDSTRING.fullmatch(document_module.file_id("データ/名前", DIST_INFO)) + + +def test_a_top_level_dist_info_is_not_the_versioned_one(): + ids = _file_ids_of(["dist-info/METADATA", "isaaccapture-0.4.0.dist-info/METADATA"]) + + assert len(set(ids.values())) == 2 + + +def test_two_files_one_id_stops_the_build(monkeypatch): + monkeypatch.setattr(document_module, "file_id", lambda path, dist_info: "x") + + with pytest.raises( + ValueError, + match=r"isaaccapture-0\.4\.0-py3-none-any\.whl: a\.py and b\.py " + r"would both be x", + ): + _file_ids_of(["a.py", "b.py"]) From 2df1b6709424a3a2cd672412bc83ea3b61b5629c Mon Sep 17 00:00:00 2001 From: Andrei Aristarkhov Date: Tue, 20 Oct 2026 16:37:06 -0700 Subject: [PATCH 093/101] refactor(sbom): name the collector package isaaccapture_sbom, keep its tests with it The collector was importable as plain `sbom`, the only generic top-level name in the repo; it is now `isaaccapture_sbom`, matching the `isaaccapture_` convention. The distribution (isaaccapture-sbom), the CLI and `uv run --project scripts/sbom isaaccapture-sbom` are unchanged. Its tests move from tests/python/sbom to scripts/sbom/tests and run from the collector's own pyproject, so the project reaches nothing through `../`: no separate tests project, no path dependency back to the collector, no repo_paths import. The repo root is found by the same VERSION + CMakeLists.txt walk, and CTest still registers them as sbom_ with the same environment, from scripts/sbom/CMakeLists.txt. uv run --project scripts/sbom --extra dev pytest scripts/sbom/tests Signed-off-by: Andrei Aristarkhov --- AGENTS.md | 8 ++-- CMakeLists.txt | 1 + {tests/python => scripts}/sbom/CMakeLists.txt | 11 ++--- .../{sbom => isaaccapture_sbom}/__init__.py | 0 .../sbom/{sbom => isaaccapture_sbom}/build.py | 0 .../sbom/{sbom => isaaccapture_sbom}/cli.py | 0 .../{sbom => isaaccapture_sbom}/discovery.py | 0 .../{sbom => isaaccapture_sbom}/document.py | 0 .../sbom/{sbom => isaaccapture_sbom}/elf.py | 0 .../{sbom => isaaccapture_sbom}/evidence.py | 0 .../{sbom => isaaccapture_sbom}/inventory.py | 0 .../{sbom => isaaccapture_sbom}/licensing.py | 0 .../{sbom => isaaccapture_sbom}/report.py | 0 .../{sbom => isaaccapture_sbom}/validate.py | 0 .../{sbom => isaaccapture_sbom}/wheelfile.py | 0 scripts/sbom/pyproject.toml | 9 +++- .../sbom => scripts/sbom/tests}/conftest.py | 47 ++++++++++--------- .../sbom => scripts/sbom/tests}/synth.py | 15 +----- .../sbom/tests}/test_discovery.py | 22 ++++----- .../sbom => scripts/sbom/tests}/test_elf.py | 4 +- .../sbom => scripts/sbom/tests}/test_gates.py | 17 +++---- .../sbom/tests}/test_licensing.py | 4 +- .../sbom/tests}/test_nested_terms.py | 12 ++--- .../sbom/tests}/test_spdx_conformance.py | 0 .../sbom/tests}/test_static_archives.py | 8 ++-- .../sbom/tests}/test_wheel_sbom.py | 17 ++++--- tests/AGENTS.md | 1 + tests/CMakeLists.txt | 2 +- tests/python/CMakeLists.txt | 3 -- tests/python/sbom/pyproject.toml | 23 --------- 30 files changed, 83 insertions(+), 121 deletions(-) rename {tests/python => scripts}/sbom/CMakeLists.txt (64%) rename scripts/sbom/{sbom => isaaccapture_sbom}/__init__.py (100%) rename scripts/sbom/{sbom => isaaccapture_sbom}/build.py (100%) rename scripts/sbom/{sbom => isaaccapture_sbom}/cli.py (100%) rename scripts/sbom/{sbom => isaaccapture_sbom}/discovery.py (100%) rename scripts/sbom/{sbom => isaaccapture_sbom}/document.py (100%) rename scripts/sbom/{sbom => isaaccapture_sbom}/elf.py (100%) rename scripts/sbom/{sbom => isaaccapture_sbom}/evidence.py (100%) rename scripts/sbom/{sbom => isaaccapture_sbom}/inventory.py (100%) rename scripts/sbom/{sbom => isaaccapture_sbom}/licensing.py (100%) rename scripts/sbom/{sbom => isaaccapture_sbom}/report.py (100%) rename scripts/sbom/{sbom => isaaccapture_sbom}/validate.py (100%) rename scripts/sbom/{sbom => isaaccapture_sbom}/wheelfile.py (100%) rename {tests/python/sbom => scripts/sbom/tests}/conftest.py (75%) rename {tests/python/sbom => scripts/sbom/tests}/synth.py (98%) rename {tests/python/sbom => scripts/sbom/tests}/test_discovery.py (99%) rename {tests/python/sbom => scripts/sbom/tests}/test_elf.py (96%) rename {tests/python/sbom => scripts/sbom/tests}/test_gates.py (98%) rename {tests/python/sbom => scripts/sbom/tests}/test_licensing.py (99%) rename {tests/python/sbom => scripts/sbom/tests}/test_nested_terms.py (98%) rename {tests/python/sbom => scripts/sbom/tests}/test_spdx_conformance.py (100%) rename {tests/python/sbom => scripts/sbom/tests}/test_static_archives.py (99%) rename {tests/python/sbom => scripts/sbom/tests}/test_wheel_sbom.py (98%) delete mode 100644 tests/python/sbom/pyproject.toml diff --git a/AGENTS.md b/AGENTS.md index cb07464b74..2ec7221a50 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -301,10 +301,10 @@ isaaccapture-sbom ...`. Its dependency pins live in `scripts/sbom/pyproject.toml and nowhere else — never restate them as `--with` flags in a workflow or a doc. **Read [`docs/source/references/sbom.rst`](docs/source/references/sbom.rst) -first** if you are changing anything under `scripts/sbom/` or -`tests/python/sbom/`, changing how a dependency reaches a wheel, or touching the -wheel-packaging steps in `build-ubuntu.yml`. It covers discovery, license -matching and the maintenance rules; nothing here repeats them. +first** if you are changing anything under `scripts/sbom/`, changing how a +dependency reaches a wheel, or touching the wheel-packaging steps in +`build-ubuntu.yml`. It covers discovery, license matching and the maintenance +rules; nothing here repeats them. ## A rule changed in one place is changed in none diff --git a/CMakeLists.txt b/CMakeLists.txt index e666d0be98..28211c5919 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -175,6 +175,7 @@ endif() # Tests (pytest + Catch2/CTest). After production libs so link targets exist. if(BUILD_TESTING) add_subdirectory(tests) + add_subdirectory(scripts/sbom) endif() # Formatting enforcement (runs on Linux by default) diff --git a/tests/python/sbom/CMakeLists.txt b/scripts/sbom/CMakeLists.txt similarity index 64% rename from tests/python/sbom/CMakeLists.txt rename to scripts/sbom/CMakeLists.txt index 0f637f3d43..c98b5dbcc9 100644 --- a/tests/python/sbom/CMakeLists.txt +++ b/scripts/sbom/CMakeLists.txt @@ -1,15 +1,11 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -# ============================================================================== -# SBOM collector tests -# ============================================================================== -# These drive synthetic wheels and build trees, so they need no built package -# and no configured dependency tree -- only the SPDX data the build fetched. +# Registers the SBOM collector's tests with CTest; they need no built package. file(GLOB TEST_FILES RELATIVE "${CMAKE_CURRENT_SOURCE_DIR}" - "${CMAKE_CURRENT_SOURCE_DIR}/test_*.py" + "${CMAKE_CURRENT_SOURCE_DIR}/tests/test_*.py" ) foreach(test_file ${TEST_FILES}) @@ -17,7 +13,8 @@ foreach(test_file ${TEST_FILES}) add_test( NAME "sbom_${test_name}" - COMMAND uv run --python ${ISAAC_TELEOP_PYTHON_VERSION} --extra dev pytest -v --tb=short "${test_file}" + COMMAND uv run --project "${CMAKE_CURRENT_SOURCE_DIR}" --python ${ISAAC_TELEOP_PYTHON_VERSION} + --extra dev pytest -v --tb=short "${test_file}" WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}" ) diff --git a/scripts/sbom/sbom/__init__.py b/scripts/sbom/isaaccapture_sbom/__init__.py similarity index 100% rename from scripts/sbom/sbom/__init__.py rename to scripts/sbom/isaaccapture_sbom/__init__.py diff --git a/scripts/sbom/sbom/build.py b/scripts/sbom/isaaccapture_sbom/build.py similarity index 100% rename from scripts/sbom/sbom/build.py rename to scripts/sbom/isaaccapture_sbom/build.py diff --git a/scripts/sbom/sbom/cli.py b/scripts/sbom/isaaccapture_sbom/cli.py similarity index 100% rename from scripts/sbom/sbom/cli.py rename to scripts/sbom/isaaccapture_sbom/cli.py diff --git a/scripts/sbom/sbom/discovery.py b/scripts/sbom/isaaccapture_sbom/discovery.py similarity index 100% rename from scripts/sbom/sbom/discovery.py rename to scripts/sbom/isaaccapture_sbom/discovery.py diff --git a/scripts/sbom/sbom/document.py b/scripts/sbom/isaaccapture_sbom/document.py similarity index 100% rename from scripts/sbom/sbom/document.py rename to scripts/sbom/isaaccapture_sbom/document.py diff --git a/scripts/sbom/sbom/elf.py b/scripts/sbom/isaaccapture_sbom/elf.py similarity index 100% rename from scripts/sbom/sbom/elf.py rename to scripts/sbom/isaaccapture_sbom/elf.py diff --git a/scripts/sbom/sbom/evidence.py b/scripts/sbom/isaaccapture_sbom/evidence.py similarity index 100% rename from scripts/sbom/sbom/evidence.py rename to scripts/sbom/isaaccapture_sbom/evidence.py diff --git a/scripts/sbom/sbom/inventory.py b/scripts/sbom/isaaccapture_sbom/inventory.py similarity index 100% rename from scripts/sbom/sbom/inventory.py rename to scripts/sbom/isaaccapture_sbom/inventory.py diff --git a/scripts/sbom/sbom/licensing.py b/scripts/sbom/isaaccapture_sbom/licensing.py similarity index 100% rename from scripts/sbom/sbom/licensing.py rename to scripts/sbom/isaaccapture_sbom/licensing.py diff --git a/scripts/sbom/sbom/report.py b/scripts/sbom/isaaccapture_sbom/report.py similarity index 100% rename from scripts/sbom/sbom/report.py rename to scripts/sbom/isaaccapture_sbom/report.py diff --git a/scripts/sbom/sbom/validate.py b/scripts/sbom/isaaccapture_sbom/validate.py similarity index 100% rename from scripts/sbom/sbom/validate.py rename to scripts/sbom/isaaccapture_sbom/validate.py diff --git a/scripts/sbom/sbom/wheelfile.py b/scripts/sbom/isaaccapture_sbom/wheelfile.py similarity index 100% rename from scripts/sbom/sbom/wheelfile.py rename to scripts/sbom/isaaccapture_sbom/wheelfile.py diff --git a/scripts/sbom/pyproject.toml b/scripts/sbom/pyproject.toml index 3563c66ee4..b4f755aa73 100644 --- a/scripts/sbom/pyproject.toml +++ b/scripts/sbom/pyproject.toml @@ -5,6 +5,7 @@ # declared once, here, rather than repeated at every call site. # # uv run --project scripts/sbom isaaccapture-sbom --help +# uv run --project scripts/sbom --extra dev pytest scripts/sbom/tests # # See docs: references/sbom. @@ -35,11 +36,15 @@ dependencies = [ dev = ["pytest", "spdx-tools==0.8.3"] [project.scripts] -isaaccapture-sbom = "sbom.cli:main" +isaaccapture-sbom = "isaaccapture_sbom.cli:main" [build-system] requires = ["setuptools>=77"] build-backend = "setuptools.build_meta" [tool.setuptools.packages.find] -include = ["sbom*"] +include = ["isaaccapture_sbom*"] + +[tool.pytest.ini_options] +testpaths = ["tests"] +pythonpath = ["tests"] # `import synth`, whatever --import-mode is diff --git a/tests/python/sbom/conftest.py b/scripts/sbom/tests/conftest.py similarity index 75% rename from tests/python/sbom/conftest.py rename to scripts/sbom/tests/conftest.py index 3245672683..9574772eff 100644 --- a/tests/python/sbom/conftest.py +++ b/scripts/sbom/tests/conftest.py @@ -10,17 +10,29 @@ import pytest import synth -from sbom import discovery, licensing - - -repo_root = synth.load( - "repo_paths", Path(__file__).resolve().parents[1] / "repo_paths.py" -).repo_root +from isaaccapture_sbom import discovery, licensing LICENSE_DATA_ENV = "ISAACCAPTURE_SBOM_LICENSE_DATA" BUILD_DIR_ENV = "ISAACCAPTURE_SBOM_BUILD_DIR" +def repo_root() -> Path | None: + """The source tree these tests sit in: the first ancestor with VERSION and CMakeLists.txt.""" + for parent in Path(__file__).resolve().parents: + if (parent / "VERSION").is_file() and (parent / "CMakeLists.txt").is_file(): + return parent + return None + + +def _candidates(env: str, *in_checkout: str): + """The environment override, then the checkout's default, asked lazily.""" + if os.environ.get(env): + yield Path(os.environ[env]) + root = repo_root() + if root is not None: + yield root.joinpath(*in_checkout) + + @pytest.fixture(scope="session") def license_data(): """The SPDX license list deps/third_party fetched into this build's _deps. @@ -29,14 +41,9 @@ def license_data(): default build directory; if neither exists the license tests skip rather than run against data that is not what the build pinned. """ - candidates = [] - if os.environ.get(LICENSE_DATA_ENV): - candidates.append(Path(os.environ[LICENSE_DATA_ENV])) - candidates.append( - repo_root() / "build" / "_deps" / "license-list-data-src" / "json" - ) - - for candidate in candidates: + for candidate in _candidates( + LICENSE_DATA_ENV, "build", "_deps", "license-list-data-src", "json" + ): try: licensing.load_corpus(candidate) return candidate @@ -73,7 +80,10 @@ def _fresh_checkout_answers(): @pytest.fixture(scope="session") def checkout(): """This repository, as the collector would be pointed at it.""" - return repo_root() + root = repo_root() + if root is None: + pytest.skip("not inside this repository's source tree") + return root @pytest.fixture(scope="session") @@ -84,12 +94,7 @@ def live_build_dir(): the anchor to what it actually emits, so a change to the codemodel's shape fails here instead of passing everywhere. """ - candidates = [] - if os.environ.get(BUILD_DIR_ENV): - candidates.append(Path(os.environ[BUILD_DIR_ENV])) - candidates.append(repo_root() / "build") - - for candidate in candidates: + for candidate in _candidates(BUILD_DIR_ENV, "build"): if sorted( (candidate / ".cmake" / "api" / "v1" / "reply").glob("codemodel-v2-*.json") ): diff --git a/tests/python/sbom/synth.py b/scripts/sbom/tests/synth.py similarity index 98% rename from tests/python/sbom/synth.py rename to scripts/sbom/tests/synth.py index f466947aa2..8e8c605843 100644 --- a/tests/python/sbom/synth.py +++ b/scripts/sbom/tests/synth.py @@ -22,7 +22,6 @@ import csv import gzip import hashlib -import importlib.util import io import os import json @@ -32,7 +31,7 @@ import zipfile from pathlib import Path -from sbom import build as build_module +from isaaccapture_sbom import build as build_module CONFIG = "Release" WHEEL_NAME = "isaaccapture-0.4.0-cp312-cp312-manylinux_2_35_x86_64.whl" @@ -176,18 +175,6 @@ def phdr(index: int, kind: int, offset: int, size: int) -> None: FIXTURE_EXCEPTIONS = ("rsync-linking-exception",) -def load(name: str, path: Path): - """Import a helper by path. - - Not via `sys.path`: this directory is named `sbom`, so putting its parent on - the path would shadow the installed collector package with a namespace one. - """ - spec = importlib.util.spec_from_file_location(name, path) - module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) - return module - - def set_license_data(path: Path) -> None: """Point the fixtures at the SPDX json/ directory the build materialized.""" global _LICENSE_DATA diff --git a/tests/python/sbom/test_discovery.py b/scripts/sbom/tests/test_discovery.py similarity index 99% rename from tests/python/sbom/test_discovery.py rename to scripts/sbom/tests/test_discovery.py index 7031b9c040..9068d6c41a 100644 --- a/tests/python/sbom/test_discovery.py +++ b/scripts/sbom/tests/test_discovery.py @@ -14,7 +14,7 @@ import pytest import synth -from sbom import discovery +from isaaccapture_sbom import discovery def test_source_trees_are_found_without_being_listed(workspace): @@ -558,7 +558,7 @@ def test_third_party_checked_in_here_is_still_third_party(workspace): into the repository that someone else wrote says so in its REUSE info, and that is what separates it from the project's own code -- not the directory it sits in.""" - from sbom import evidence as evidence_module + from isaaccapture_sbom import evidence as evidence_module space = evidence_module.discover(workspace.root, workspace.build) vendored = { @@ -586,7 +586,7 @@ def test_third_party_checked_in_here_is_still_third_party(workspace): def test_a_vendored_header_is_credited_to_the_artifact_that_includes_it(workspace): - from sbom import evidence as evidence_module + from isaaccapture_sbom import evidence as evidence_module space = evidence_module.discover(workspace.root, workspace.build) artifact = space.graph.by_name("_ext.cpython-312-x86_64-linux-gnu.so")[0] @@ -631,7 +631,7 @@ def test_a_second_build_tree_is_not_mistaken_for_source(workspace, license_data) an unexplained wheel member rather than a wrong one. A configured tree says what it is with a CMakeCache.txt. """ - from sbom import evidence, licensing + from isaaccapture_sbom import evidence, licensing licensing.load_corpus(license_data) stray = workspace.root / "build-debug" @@ -656,7 +656,7 @@ def test_a_fetched_archive_records_where_it_came_from(workspace, license_data): rather than guessed; without that file the document must say it does not know. """ - from sbom import evidence, licensing + from isaaccapture_sbom import evidence, licensing licensing.load_corpus(license_data) archive = workspace.root / synth.SDK_ARCHIVE @@ -682,7 +682,7 @@ def test_a_fetched_archive_records_where_it_came_from(workspace, license_data): def test_a_build_tree_with_no_configuration_says_so(workspace): """Guessing Release would read a directory that need not exist, and every member it would have explained becomes an unexplained one instead.""" - from sbom import evidence + from isaaccapture_sbom import evidence cache = workspace.build / "CMakeCache.txt" cache.write_text( @@ -700,7 +700,6 @@ def test_a_build_tree_with_no_configuration_says_so(workspace): def test_the_supplier_is_read_from_the_project_not_written_here(workspace): """A fork or a rename must not leave the document naming the wrong party.""" - from sbom import discovery assert discovery.project_supplier(workspace.root) == "Organization: NVIDIA" @@ -717,7 +716,6 @@ def test_the_supplier_is_read_from_the_project_not_written_here(workspace): def test_a_project_naming_several_authors_stops_rather_than_choosing(workspace): """SPDX records one supplier; which one is not this tool's call to make.""" - from sbom import discovery pyproject = workspace.root / "pyproject.toml" pyproject.write_text( @@ -742,7 +740,7 @@ def test_a_relative_build_dir_still_keeps_the_staged_tree_out( member has to pass. The exclusion compared an absolute skip path against relative walked ones and silently matched nothing. """ - from sbom import evidence, licensing + from isaaccapture_sbom import evidence, licensing licensing.load_corpus(license_data) monkeypatch.chdir(workspace.root) @@ -764,7 +762,7 @@ def test_a_dependency_installed_into_the_build_tree_is_not_vendored_source( of them on a real build -- each then failing the gate for shipping no licence text of its own. """ - from sbom import evidence, licensing + from isaaccapture_sbom import evidence, licensing licensing.load_corpus(license_data) port = workspace.build / "vcpkg_installed/x64-linux/include/eigen/Core.h" @@ -784,7 +782,7 @@ def test_vendored_material_asserts_no_package_identifier(workspace, license_data `pkg:generic/` from it named a package that does not exist, and a reader resolving that locator finds nothing or finds something else. """ - from sbom import discovery, licensing + from isaaccapture_sbom import licensing licensing.load_corpus(license_data) source = workspace.root / "src" / "vendor" / "widget.hpp" @@ -1245,7 +1243,7 @@ def test_the_files_the_build_read_are_named_first(tmp_path): def test_a_tracked_non_code_file_reaches_discovery(workspace, license_data): """Through the build's own entry point, not only `discover_vendored`.""" - from sbom import evidence, licensing + from isaaccapture_sbom import evidence, licensing licensing.load_corpus(license_data) synth.write(workspace.root / "assets/vendor/icon.svg", "\n") diff --git a/tests/python/sbom/test_elf.py b/scripts/sbom/tests/test_elf.py similarity index 96% rename from tests/python/sbom/test_elf.py rename to scripts/sbom/tests/test_elf.py index 412021a95d..25e22dcb3c 100644 --- a/tests/python/sbom/test_elf.py +++ b/scripts/sbom/tests/test_elf.py @@ -7,7 +7,7 @@ import pytest import synth -from sbom import elf +from isaaccapture_sbom import elf def test_reads_soname_and_needed(): @@ -59,7 +59,7 @@ def test_reads_a_real_system_library(): def test_verification_does_not_parse_binaries(built): """A consumer checking a downloaded wheel should not run a binary parser over it; scanning only reads the dynamic section when asked to.""" - from sbom import wheelfile + from isaaccapture_sbom import wheelfile plain = wheelfile.scan(built["wheel"]) analyzed = wheelfile.scan(built["wheel"], analyze_elf=True) diff --git a/tests/python/sbom/test_gates.py b/scripts/sbom/tests/test_gates.py similarity index 98% rename from tests/python/sbom/test_gates.py rename to scripts/sbom/tests/test_gates.py index 2f349c9e57..41957ea362 100644 --- a/tests/python/sbom/test_gates.py +++ b/scripts/sbom/tests/test_gates.py @@ -12,11 +12,11 @@ import pytest import synth -from sbom import build as build_module -from sbom import cli -from sbom import licensing -from sbom import wheelfile -from sbom import validate as validate_module +from isaaccapture_sbom import build as build_module +from isaaccapture_sbom import cli +from isaaccapture_sbom import licensing +from isaaccapture_sbom import wheelfile +from isaaccapture_sbom import validate as validate_module DIST_INFO = synth.DIST_INFO WHEEL_NAME = synth.WHEEL_NAME @@ -523,7 +523,6 @@ def test_a_name_match_a_build_id_contradicts_is_refused(workspace, license_data) on -- and a differing build-id says it is the wrong one. Claiming it would hand this binary the other component's licence. """ - from sbom import licensing licensing.load_corpus(license_data) payload = synth.wheel_payload(workspace) @@ -548,7 +547,7 @@ def test_a_name_match_a_build_id_contradicts_is_refused(workspace, license_data) def test_an_unreadable_archive_says_so(workspace, license_data): """Only archives this build fetched are indexed, so one that will not open is a broken download -- not a stray file to walk past.""" - from sbom import discovery, licensing + from isaaccapture_sbom import discovery licensing.load_corpus(license_data) (workspace.root / synth.SDK_ARCHIVE).write_bytes(b"not an archive at all") @@ -571,7 +570,6 @@ def test_bytes_more_than_one_component_holds_name_no_single_source( obligation under another's. Say the bytes are shared and claim no single source instead. """ - from sbom import licensing licensing.load_corpus(license_data) shared = synth.spdx_text("Apache-2.0").encode() @@ -659,7 +657,7 @@ def test_one_discovery_serves_every_wheel_without_bleeding_between_them( document names licence texts the evidence cannot account for and the wheel fails its own verification. """ - from sbom import evidence as evidence_module + from isaaccapture_sbom import evidence as evidence_module copyright_file = tmp_path / "copyright" copyright_file.write_text(synth.spdx_text("Zlib"), encoding="utf-8") @@ -712,7 +710,6 @@ def test_a_third_partys_licence_does_not_discharge_this_one(workspace, license_d a wheel that stopped shipping its own LICENSE still passed and the only copy of its terms sat under `third-party/`, attributed to somebody else. """ - from sbom import build as build_module licensing.load_corpus(license_data) assert build_module._own_license_texts( diff --git a/tests/python/sbom/test_licensing.py b/scripts/sbom/tests/test_licensing.py similarity index 99% rename from tests/python/sbom/test_licensing.py rename to scripts/sbom/tests/test_licensing.py index 8d4baff6b7..e6d2769ea7 100644 --- a/tests/python/sbom/test_licensing.py +++ b/scripts/sbom/tests/test_licensing.py @@ -10,8 +10,7 @@ import pytest import synth -from sbom import discovery, document, report -from sbom import licensing +from isaaccapture_sbom import discovery, document, licensing, report def test_corpus_loads(): @@ -774,7 +773,6 @@ def test_a_choice_of_licences_is_not_published_as_both(license_data, tmp_path): name loses the operator, and the roll-up that splits an expression into identifiers loses it again. """ - from sbom import discovery licensing.load_corpus(license_data) source = tmp_path / "vendored.h" diff --git a/tests/python/sbom/test_nested_terms.py b/scripts/sbom/tests/test_nested_terms.py similarity index 98% rename from tests/python/sbom/test_nested_terms.py rename to scripts/sbom/tests/test_nested_terms.py index bcd553b0c0..c860f262b8 100644 --- a/tests/python/sbom/test_nested_terms.py +++ b/scripts/sbom/tests/test_nested_terms.py @@ -11,12 +11,12 @@ import pytest import synth -from sbom import build as build_module -from sbom import discovery -from sbom import evidence as evidence_module -from sbom import inventory as inventory_module -from sbom import licensing -from sbom import report +from isaaccapture_sbom import build as build_module +from isaaccapture_sbom import discovery +from isaaccapture_sbom import evidence as evidence_module +from isaaccapture_sbom import inventory as inventory_module +from isaaccapture_sbom import licensing +from isaaccapture_sbom import report BUNDLED = "Copyright (c) 2019 Bundled Authors\n\n" + "Permission is hereby granted." EXTENSION = "isaaccapture/_ext.cpython-312-x86_64-linux-gnu.so" diff --git a/tests/python/sbom/test_spdx_conformance.py b/scripts/sbom/tests/test_spdx_conformance.py similarity index 100% rename from tests/python/sbom/test_spdx_conformance.py rename to scripts/sbom/tests/test_spdx_conformance.py diff --git a/tests/python/sbom/test_static_archives.py b/scripts/sbom/tests/test_static_archives.py similarity index 99% rename from tests/python/sbom/test_static_archives.py rename to scripts/sbom/tests/test_static_archives.py index 72e7c420a4..4ec5d487be 100644 --- a/tests/python/sbom/test_static_archives.py +++ b/scripts/sbom/tests/test_static_archives.py @@ -10,10 +10,10 @@ import pytest import synth from packageurl import PackageURL -from sbom import build as build_module -from sbom import discovery -from sbom import document as document_module -from sbom import evidence as evidence_module +from isaaccapture_sbom import build as build_module +from isaaccapture_sbom import discovery +from isaaccapture_sbom import document as document_module +from isaaccapture_sbom import evidence as evidence_module EXTENSION = "_ext.cpython-312-x86_64-linux-gnu.so" ONE_OBJECT = synth.ar_archive({"runtime.o": b"\x7fELF object"}) diff --git a/tests/python/sbom/test_wheel_sbom.py b/scripts/sbom/tests/test_wheel_sbom.py similarity index 98% rename from tests/python/sbom/test_wheel_sbom.py rename to scripts/sbom/tests/test_wheel_sbom.py index 627dec7144..c85ecb86bc 100644 --- a/tests/python/sbom/test_wheel_sbom.py +++ b/scripts/sbom/tests/test_wheel_sbom.py @@ -12,13 +12,13 @@ import pytest import synth -from sbom import build as build_module -from sbom import document as document_module -from sbom import inventory as inventory_module -from sbom import licensing -from sbom import report as report_module -from sbom import validate as validate_module -from sbom import wheelfile +from isaaccapture_sbom import build as build_module +from isaaccapture_sbom import document as document_module +from isaaccapture_sbom import inventory as inventory_module +from isaaccapture_sbom import licensing +from isaaccapture_sbom import report as report_module +from isaaccapture_sbom import validate as validate_module +from isaaccapture_sbom import wheelfile DIST_INFO = synth.DIST_INFO WHEEL_NAME = synth.WHEEL_NAME @@ -426,7 +426,6 @@ def test_source_date_epoch_normalizes_timestamps_the_input_wheel_carried( @pytest.fixture def shared_text(workspace, license_data): """alpha and beta ship the demo SDK's proprietary terms byte for byte.""" - from sbom import licensing licensing.load_corpus(license_data) for key in ("alpha", "beta"): @@ -498,7 +497,7 @@ def test_a_requirement_this_run_also_built_is_not_an_unknown( only one direction could ever carry such a reference, and a `py3-none-any` wheel pinning one matrix entry's sibling stops being universal. """ - from sbom import cli + from isaaccapture_sbom import cli synth.write_wheel(workspace.wheel, synth.wheel_payload(workspace)) companion = workspace.wheel.parent / "isaacteleop-0.4.0-py3-none-any.whl" diff --git a/tests/AGENTS.md b/tests/AGENTS.md index 98c75f7bc3..e777b9c8f5 100644 --- a/tests/AGENTS.md +++ b/tests/AGENTS.md @@ -75,3 +75,4 @@ Shared C++ fixtures (not executables) live under `tests/cpp/viz/support/` - `src/core/codegen/test_*.py` — unittest, co-located with the generator. - `examples/oxr/python/test_*.py` — standalone scripts for CloudXR GPU CI. - ROS2 Docker integration helpers under `examples/teleop_ros2/.../integration_tests/`. +- `scripts/sbom/tests/` — the SBOM collector's tests, run from its own uv project. diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 59e4ea5dc6..84bd3a5aa1 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -5,7 +5,7 @@ # IsaacTeleop tests (pytest + Catch2/CTest) # ============================================================================== # -# Production libraries live under src/ and examples/; all pytest and Catch2 +# Production libraries live under src/ and examples/; pytest and Catch2 # suites live here under tests/python/ and tests/cpp/. if(BUILD_TESTING) diff --git a/tests/python/CMakeLists.txt b/tests/python/CMakeLists.txt index f58f95c936..f2609a4e58 100644 --- a/tests/python/CMakeLists.txt +++ b/tests/python/CMakeLists.txt @@ -14,9 +14,6 @@ if(BUILD_PYTHON_BINDINGS) add_subdirectory(compat) endif() -# Tooling tests: synthetic fixtures, so no built package is needed. -add_subdirectory(sbom) - if(BUILD_EXAMPLES AND BUILD_PYTHON_BINDINGS) add_subdirectory(examples/teleop) endif() diff --git a/tests/python/sbom/pyproject.toml b/tests/python/sbom/pyproject.toml deleted file mode 100644 index 0bf1586bc1..0000000000 --- a/tests/python/sbom/pyproject.toml +++ /dev/null @@ -1,23 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -[project] -name = "isaaccapture-sbom-tests" -version = "0.0.0" # Internal tests - not versioned -description = "Tests for the wheel SBOM and license evidence collector" -requires-python = ">=3.11,<3.14" - -[project.optional-dependencies] -# The collector itself, from the source tree: its pins live in its own -# pyproject, so there is one place to bump them. -dev = ["isaaccapture-sbom[dev]"] - -[tool.uv.sources] -isaaccapture-sbom = { path = "../../../scripts/sbom", editable = true } - -[tool.pytest.ini_options] -pythonpath = ["."] -python_files = ["test_*.py"] -python_classes = ["Test*"] -python_functions = ["test_*"] -norecursedirs = [".git", ".venv", "build", "dist", "*.egg", "__pycache__"] From 7f8013e1ba93858222a18b93381f3dacc5b4bd1c Mon Sep 17 00:00:00 2001 From: Andrei Aristarkhov Date: Tue, 20 Oct 2026 17:05:34 -0700 Subject: [PATCH 094/101] docs(sbom): a short page for wheel users, a README for the collector docs/source/references/sbom.rst had grown into a maintainer reference few people would read through. It is now the page a wheel consumer needs: what a wheel carries, what a release publishes beside it, and how to verify either. How the collector works moves to scripts/sbom/README.md, next to the code: running and testing it, how the inventory and licenses are worked out, and what stops the Release build, each condition with its remedy. The two link to each other and state each fact once. Root AGENTS.md points agents at the README with one rule (fix the cause; never exclude a file or record an assumed license), and every build error that stops a Release build names that README section. Also corrected where the split exposed them: license.rst no longer says every SPDX header must be Apache-2.0 (checked-in third-party files carry upstream's terms, EgoPost its own), and the build prerequisites say a Release wheel with its SBOM needs CMake 3.27. Signed-off-by: Andrei Aristarkhov --- AGENTS.md | 21 +- deps/README.md | 2 +- .../development/build_from_source/index.rst | 3 +- docs/source/development/license.rst | 5 +- docs/source/references/sbom.rst | 485 ++---------------- scripts/sbom/README.md | 387 ++++++++++++++ scripts/sbom/isaaccapture_sbom/build.py | 18 +- scripts/sbom/isaaccapture_sbom/discovery.py | 5 +- scripts/sbom/pyproject.toml | 2 +- 9 files changed, 469 insertions(+), 459 deletions(-) create mode 100644 scripts/sbom/README.md diff --git a/AGENTS.md b/AGENTS.md index 2ec7221a50..b324e6b5c1 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -285,26 +285,17 @@ build itself. There is **no checked-in dependency registry** and adding one woul be a second source of truth — declare a dependency the way you already would and it appears in the next wheel's inventory. -These fail the Release build rather than shipping a gap, and none is fixed by -excluding a file or writing down an assumption: - -- **a wheel member the build cannot explain** — teach the collector that route; -- **static code on a link line the collector cannot trace** — from the build - machine, or installed into the build tree by ExternalProject; link a - traceable copy, or teach the collector that route; -- **a redistributed component with no license text** — obtain the terms; -- **a checked-in third-party file that fails the per-file REUSE gate** — record - upstream's terms in its REUSE info (sbom.rst says how). +When the SBOM step stops the Release build, fix the cause; never exclude a file or +record an assumed license. Remedies: `scripts/sbom/README.md` ('What stops the Release +build'). The collector is a project of its own: `uv run --project scripts/sbom isaaccapture-sbom ...`. Its dependency pins live in `scripts/sbom/pyproject.toml` and nowhere else — never restate them as `--with` flags in a workflow or a doc. -**Read [`docs/source/references/sbom.rst`](docs/source/references/sbom.rst) -first** if you are changing anything under `scripts/sbom/`, changing how a -dependency reaches a wheel, or touching the wheel-packaging steps in -`build-ubuntu.yml`. It covers discovery, license matching and the maintenance -rules; nothing here repeats them. +**Read [`scripts/sbom/README.md`](scripts/sbom/README.md) first** if you are +changing anything under `scripts/sbom/`, changing how a dependency reaches a +wheel, or touching the wheel-packaging steps in `build-ubuntu.yml`. ## A rule changed in one place is changed in none diff --git a/deps/README.md b/deps/README.md index 793a10c26a..9ada231254 100644 --- a/deps/README.md +++ b/deps/README.md @@ -112,7 +112,7 @@ machine and are located via `find_package`. - **Version**: 31ba1a50e5397e00a304dbadc76531740e89ee48 (v3.29.0) - **Purpose**: reference license texts. The SBOM collector names a dependency's license by matching the text that dependency ships against these; nothing from - this checkout is redistributed. See `docs/source/references/sbom.rst`. + this checkout is redistributed. See `scripts/sbom/README.md`. - **Build**: data only, no build step. - **License**: CC0-1.0 diff --git a/docs/source/development/build_from_source/index.rst b/docs/source/development/build_from_source/index.rst index f62565738a..c3202c792b 100644 --- a/docs/source/development/build_from_source/index.rst +++ b/docs/source/development/build_from_source/index.rst @@ -20,7 +20,8 @@ Prerequisites ------------- - **CMake** 3.24 or higher (Ubuntu 22.04's apt ``cmake`` is 3.22 — install a newer one - from `Kitware's APT repository `_ or with ``pip install cmake``) + from `Kitware's APT repository `_ or with ``pip install cmake``). + A Release wheel with its contents SBOM needs 3.27, which added ``cmake_file_api()``. - **C++20** compatible compiler - **Python** 3.11, 3.12, or 3.13 (default 3.11; see ``ISAAC_TELEOP_PYTHON_VERSION`` in root ``CMakeLists.txt``) - **uv** for Python dependency management and managed Python diff --git a/docs/source/development/license.rst b/docs/source/development/license.rst index 62a40de509..f8f8a26587 100644 --- a/docs/source/development/license.rst +++ b/docs/source/development/license.rst @@ -25,7 +25,10 @@ We use `REUSE `_ to enforce SPDX license headers on sou ``reuse lint-file`` on changed files (e.g. ``.py``, ``.cpp``, ``.md``, ``.yaml``): 1. **Required fields**: Each file must have ``SPDX-FileCopyrightText`` and ``SPDX-License-Identifier``. -2. **License**: ``SPDX-License-Identifier`` must be ``Apache-2.0`` (full text in ``LICENSES/Apache-2.0.txt``). +2. **License**: first-party files use ``Apache-2.0`` (full text in ``LICENSES/Apache-2.0.txt``), + except EgoPost (``src/python/isaaccapture_egopost/``), which uses ``LicenseRef-OpenMDW-1.1``. + Checked-in third-party files record upstream's terms instead; see *What stops the Release + build* in :code-file:`scripts/sbom/README.md`. Example header (for Python files): diff --git a/docs/source/references/sbom.rst b/docs/source/references/sbom.rst index 54c37da05d..2273ecf8d0 100644 --- a/docs/source/references/sbom.rst +++ b/docs/source/references/sbom.rst @@ -4,9 +4,12 @@ Wheel contents SBOM and license evidence ======================================== -Every published ``isaaccapture`` wheel carries a machine-readable inventory of what it -redistributes, plus the license text behind each obligation. You can inspect both -without installing the package, running it, or contacting a hosted service. +Every published ``isaaccapture`` wheel lists what it redistributes and carries the license +text behind each obligation. You can check both offline, without installing the wheel. The +Release build refuses to ship a wheel with a gap in either. + +To check a wheel, see `Verifying a wheel`_. To learn how the evidence is worked out, or why +a build stopped, see the collector's developer README, :code-file:`scripts/sbom/README.md`. What a wheel carries -------------------- @@ -18,46 +21,51 @@ What a wheel carries ├── METADATA # License-Expression + one License-File per text below ├── licenses/ │ ├── LICENSE.md # this project's own license - │ ├── LICENSES/.txt # the pool texts license-files in pyproject.toml names - │ ├── THIRD-PARTY-NOTICES.md # component -> license -> text, with roles - │ └── third-party// # its own file verbatim, or a reference text, labelled + │ ├── LICENSES/.txt # this project's license texts + │ ├── THIRD-PARTY-NOTICES.md # each component, its license, and where the text is + │ └── third-party// # the component's own license file, or a reference text └── sboms/ └── .spdx.json # SPDX 2.3 contents inventory -Every text there comes from this repository, or from a component the wheel redistributes, -or — for material checked into this repository, whose terms its REUSE metadata declares — -is the reference text each declared identifier names, beside the copyright notice it is -declared under. See `How licenses are determined`_. +- **The SBOM** lists every file in the wheel but ``RECORD`` and itself: where it came from, + under what license, and its digest. It describes the wheel's own contents only. Build + tools (listed in the report under *Present in this build but not in this wheel*), + install-time dependencies (``Requires-Dist`` stays authoritative; the SBOM records the + requirement, never a license for it), source distributions and ``pip install .``, + containers and the web client are not covered. +- **Each third-party component** ships its own license file or, for material checked into + this repository, the reference text for its declared license beside its copyright + notice. +- ``sboms/`` may also hold a CycloneDX document ``auditwheel`` writes for the libraries it + bundled. + +Evidence published with a release +--------------------------------- -``.dist-info/sboms/`` is what the -`binary distribution format `_ -reserves for SBOM documents, and it is shared: ``auditwheel`` writes a CycloneDX document -of its own there describing the libraries it vendored. This collector looks for its -document by name and leaves anything else alone. Reports and manifests are published beside -the wheel instead. +Each tagged GitHub release carries, for every wheel: -Verifying a wheel ------------------ +``.spdx.json`` + An exact copy of the document embedded in the wheel. -Integrity and provenance are the packaging ecosystem's job, not this document's: +``.licenses.md`` / ``.licenses.json`` + Each redistributed component, the license concluded for it and the file it was read + from, how it reaches the wheel, what the installation environment supplies instead, and + what was excluded and why. -``RECORD`` - Every wheel already carries a ``sha256`` and size for each of its files, and - installers check them. That is the integrity mechanism for a wheel's contents. +``.build-evidence.json`` + Resolved commits, declared constraints, archive digests and build options, from the + build machine. -`PEP 740 `_ attestations - Signed attestations binding a publisher identity to an artifact's hash, served - through PyPI's Integrity API. That is the ecosystem's mechanism for binding - provenance to a distribution; these releases are not published with attestations. +and, once per release: -The SBOM *describes* what those files are — where each came from, under what license. -Its per-file digests are there so the description can be shown to belong to the wheel in -front of you, and so it can be checked against ``RECORD``; they are not a second root of -trust. +``isaaccapture-sbom-manifest.json`` + Each wheel's filename and SHA-256 and its evidence files' digests, recorded after the + last change to the wheel, so a wheel touched since no longer matches. -The document lists every member of the wheel except two: ``RECORD`` and the document -itself. Neither can carry its own digest, since ``RECORD`` hashes the document and the -document would hash ``RECORD``. ``check`` reads both against the archive directly. +Verifying a wheel +----------------- + +You need ``uv`` and a clone of this repository: .. code-block:: bash @@ -65,59 +73,8 @@ document would hash ``RECORD``. ``check`` reads both against the archive directl uv run --project IsaacCapture/scripts/sbom isaaccapture-sbom check \ --wheel isaaccapture---.whl -The collector is a project of its own, :code-file:`scripts/sbom/pyproject.toml`, so its -dependencies come with it and are pinned there rather than at each call site. They are the -packages that own the formats involved — ``installer`` for ``RECORD``, ``packaging`` for -core metadata, ``license-expression`` for SPDX expressions, ``packageurl-python`` for -purls, ``pyelftools`` for ELF, ``reuse`` for REUSE metadata. ``reuse`` reads files through -libmagic, so ``build`` needs it (``libmagic1`` on Ubuntu); ``check`` does not. - -One command, widening with whatever you have. With just a wheel it confirms that - -- the SBOM covers every file in the wheel except ``RECORD`` and the SBOM itself, -- each digest the SBOM records matches the bytes present, and agrees with ``RECORD``, -- ``RECORD`` itself agrees with the archive, so the wheel is still installable, -- the packaged notices are there, and every ``LicenseRef-`` has its text embedded. - -Add ``--build-evidence`` and it also checks that the build accounts for every member; -add ``--manifest`` and it checks the wheel against the digest bound at publication and -the sidecars beside it. ``check-set`` runs the whole thing over every wheel in a release. - -To read the inventory directly, unzip ``.dist-info/sboms/*.spdx.json`` and feed it to any -SPDX 2.3 consumer. CI validates each published document with ``spdx-tools`` before release. - -A file's ``SPDXID`` is its path in the distribution, reduced to the characters an SPDX -identifier allows, plus a short digest of the unreduced path, with the versioned -``.dist-info`` and ``.data`` directories counted as fixed. A file keeps its -identifier across releases unless that path changes, so two releases' documents can be -diffed. - -Evidence published beside a wheel ---------------------------------- - -Tagged releases attach the following to the GitHub release, so the evidence outlives the -CI artifacts it travelled in: - -``.spdx.json`` - Byte-identical copy of the document embedded in that wheel. - -``.licenses.md`` / ``.licenses.json`` - Component-to-license report: every redistributed component, the SPDX expression - concluded for it, the file that expression was read from, how the component reaches - the wheel, what is supplied by the installation environment instead, and what is - excluded and why. - -``.build-evidence.json`` - Resolved commits, declared constraints, archive digests and build options recorded on - the machine that built the wheel. - -``isaaccapture-sbom-manifest.json`` - Maps each published wheel filename and SHA-256 to its evidence files and their digests. - The wheel digest is bound here, after the last change to the wheel, so a wheel that was - touched again no longer matches. - -Downloading a whole release? Put its wheels and evidence files in a directory of their -own, then check them against the manifest in one step: +To check a whole release, put its wheels and evidence files in a directory of their own, +and keep other wheels out of it: .. code-block:: bash @@ -125,352 +82,14 @@ own, then check them against the manifest in one step: --manifest release/isaaccapture-sbom-manifest.json \ --wheel-dir release --evidence-dir release -Any command reading a manifest refuses one that lists no wheel. ``check-set`` also fails a -wheel in ``--wheel-dir`` that the manifest does not list, which is why the directory holds -the release and nothing else. - -What is in scope ----------------- - -The document describes **the contents of one built wheel**: compiled-in static and -header-only dependencies, bundled shared libraries, vendored sources, SDK binaries and -packaged assets, for all six published variants (``x86_64`` and ``aarch64`` × CPython -3.11, 3.12, 3.13). - -Deliberately **not** described by it: - -- Build-time-only inputs — test frameworks, code generators, CMake modules, toolchains. - They appear in the report under *Present in this build but not in this wheel*; a full - build inventory is separate work. -- Source distributions and the ``pip install .`` path, which produce different contents. -- Containers and the web client, which are distributed on their own. -- Dependencies resolved at install time. The wheel's ``Requires-Dist`` metadata stays - authoritative for those; the document records the requirement, never a license for it. - -How the inventory is worked out -------------------------------- - -There is no checked-in dependency list. Every entry is derived from the build that -produced the wheel, so a dependency that builds is a dependency that appears: - -``build/_deps/*-src`` - FetchContent trees. ``git`` in each checkout supplies the commit, the remote, and from - the remote the supplier, homepage and purl. It is asked only of a tree that is a - repository of its own: git otherwise answers for whichever checkout encloses the build - directory, or for whatever repository ``GIT_DIR`` names, which is therefore cleared. Any - other tree is identified by its declaration — ``GIT_REPOSITORY`` and ``GIT_TAG``, or - ``URL`` and ``URL_HASH`` — from the stamps ExternalProject writes beside the populate - sub-build, or under ``CMakeFiles/fc-stamp`` once policy ``CMP0168`` is ``NEW``, and - from the clone script the git stamp names. A stamp outlives a change of method, so where - both are present the tree decides: a git clone is a checkout of its own. The version is - the declared tag, or the ref a GitHub URL names, verbatim, so a tag gives one version - whether declared by tag or as a GitHub archive; any other archive's version is read - from its name, and its supplier is not asserted, since the host serving a download is - not who supplies it. A download location is stated only for an ``http``, ``https``, - ``ftp`` or ``ftps`` archive URL, or a git remote reached over ``http``, ``https``, - ``ssh`` or ``git``, and never with its credentials, query or fragment. - -vcpkg's install records - Where the build uses vcpkg, the ``vcpkg/info`` lists under ``VCPKG_INSTALLED_DIR`` (by - default the build tree's ``vcpkg_installed``): which port installed each file. - -CMake's file API codemodel - The build graph, as CMake itself describes it: which sources were compiled into which - binary, which targets it depends on, what its link line names, and — - through include paths — which header-only dependencies it was compiled against. - Followed transitively, so a library linked into a library linked into the wheel is still - reported. The top-level ``CMakeLists.txt`` asks - for it with ``cmake_file_api()``, which CMake 3.27 added — so a wheel with an SBOM - needs at least that, though the project builds without it. Reading the generator's own - output instead would only work under Makefiles, since Ninja writes no ``link.txt``. - -The compiler's dependency output - Which headers each object read: the per-object depfiles and ``compiler_depend.make`` - under the Makefile generators, ``ninja -t deps`` under Ninja — so a Ninja build is - collected where ``ninja`` is on ``PATH``, and the collector says so once when it is - not. Any target whose records do not cover every source it compiles falls back to - what it lists and includes. - -Archives in the repository - Every member of every tarball is hashed. A wheel member that matches one is recorded as - having been unpacked from that archive. - -The working tree and the build tree - What remains is matched against the project's own sources and its generated output. - -**Every member of the wheel must be explained by one of those.** A file that is not stops -the build rather than being quietly omitted — that is the check that keeps the inventory -honest as the build changes. - -Post-processing is recorded as such. ``patchelf``, ``auditwheel``'s RPATH rewrite and the -build's MJCF asset stripping all leave bytes that no longer hash to their origin, so those -members are reported as *derived from* a named input rather than as an exact copy, with -the input's digest wherever the input was hashed — which it is for an archive member, and -is not for a file matched only by its path. - -How licenses are determined ---------------------------- - -Licenses are never inferred from this project's license, and never read from a list someone -maintained by hand. REUSE metadata is not such a list: it is how this repository declares -the terms of each file it holds — an in-file ``SPDX-License-Identifier``, a ``.license`` -sidecar or a ``REUSE.toml`` annotation, one declaration — and the pre-commit hook lints it -for source files. The collector reads it for every file here that REUSE covers, whatever its -type, and for every source here the build compiled or included: a file identified as a third -party's (see below) is held to the per-file gate even where the hook does not lint it, and a -file naming no holder at all counts as this project's own. A fetched component's licence is -not read from its source headers, ``.license`` sidecars or a ``REUSE.toml``: for each -fetched tree the collector finds the license and notice files the component itself ships — -``LICENSE``, ``COPYING``, ``BSD-LICENSE``, a ``LICENSES/`` pool, or, where none of those at -its root is a grant, the license section of its README — and matches each text against a -corpus of SPDX reference texts. An archive in the repository is read only for the license -files at its root. - -Those reference texts are a dependency like any other. :code-file:`deps/third_party/CMakeLists.txt` -fetches -``spdx/license-list-data`` at a pinned commit into ``_deps/license-list-data-src``, the same -mechanism and the same place as every other dependency — which means the collector -discovers it as a component of the build and reports it, alongside the rest, as fetched but -shipping nothing. Pinning is by commit, so no tag or branch upstream can move underneath a -build, and each report records the commit its identifications were made against. - -Consulting the list is not shipping it. Every current identifier is available to match -against, so a dependency arriving under any of them is identified. Naming it in the -document is a second question: a document states the licence list it was built against, and -``license-expression`` — which a consumer validates with, and which -:code-file:`scripts/sbom/pyproject.toml` pins — ships its own, older copy. An id the fetched -list knows and that grammar does not cannot be published as an id, so the text it was read -from travels as a ``LicenseRef-`` — a REUSE declaration naming one travels whole, since an -``OR`` or a ``WITH`` cannot be kept in part — and the component is listed under *License -evidence gaps* with that as the stated reason. - -What the wheel carries for a third-party component is that component's own license file, -verbatim, copyright holders and all. Substituting a reference text would convey the terms -and drop the notice — for MIT and the BSD licenses those are the same file, differing by -the one line that names who licensed it to you — and it would replace the component's -actual grant with our reading of it, which is what the matching is for and not what it -decides. So a fetched component ships its own text, or, having none, blocks the build -(below). - -Third-party material checked into this repository rather than fetched is the exception: it -is identified by the copyright holder it is declared under — a ``.license`` sidecar or a -``REUSE.toml`` annotation where REUSE applies one, and the first copyright line of the file -itself unless REUSE applies those in the file's place — and its terms are the identifiers -REUSE gives each of its files, under REUSE's own precedence. The reference text for each -licence it names ships, and the expression keeps its structure, so a choice stays a choice; -an exception's text ships too where the expression travels as a ``LicenseRef-``, above. The -document says so for every such text, in as many words, because a reader cannot tell a -template from an author's own notice by looking at it. - -A component can also state terms for one part of itself — a vendored asset set or a bundled -library inside a larger project, with its own ``LICENSE`` and its own holder. Those are read -and packaged too, but only where something under that directory reached the wheel: a file -copied or derived from it — a copy the build made into the component's own build tree -included — a source compiled from it, or a header the compiler's dependency output says it -read. Where a target compiles something its dependency output does not account for, the -sources it lists count instead, and so does any include directory above or below that -directory. An archive of the component linked in keeps all of them, since what it holds is -unknown. A REUSE ``.license`` sidecar covers only the file it annotates. Which of those kept -each text is in the build evidence. - -A bundled library compiled in ships its texts and holders, each listed as terms stated for -part of the component; the component's concluded licence stays its own grant. In the evidence -listed for a text, one that matches several identifiers sharing one reference text — -``GPL-3.0-only`` and ``GPL-3.0-or-later`` — is listed as matching either, not both. - -The distribution's own declared licence is an obligation like any other. A wheel declaring -``License-Expression`` and shipping no text for it stops the build, the same as a -redistributed component with no terms; where the wheel ships none, the reference text for -each identifier is packaged from the pinned list and declared under PEP 639. - -The notices file opens with the distribution's own licence and copyright holder, read -from the packaging metadata it was built from where no file it ships states one. A wheel -that ships only metadata carries no notice otherwise: the reference text its declared -licence packages is a template naming nobody, so a recipient holding the wheel alone could -not say who licensed it to them. - -``licenseDeclared`` on the wheel is what the distribution says of itself. ``licenseConcluded`` -is what this document found in it, so a wheel carrying a component whose terms the build -could not name states that component's ``LicenseRef-`` alongside the declaration — a reader -asking "what is this artifact" is told about the proprietary payload rather than about the -declaration only. Where nothing at all is concluded for a component, neither is anything -for the wheel: it states ``NOASSERTION``. - -Never a file from this checkout. ``LICENSES/`` holds this repository's REUSE pool, whose -copies exist to license *this* project and carry its notices; packaging one as a third -party's terms would put our copyright above their code. A substituted text comes from the -pinned list, never from the pool — and under ``.dist-info/licenses/LICENSES/`` a wheel -carries whichever applies: the project's own pool files where it has them, and otherwise -the reference text for the licence it declares. - -A match needs a canonical license text to be almost entirely present *and* to account for a -real share of the file: a tenth of a long agreement reproducing a license in an appendix is -not that agreement's own terms. The second threshold is the looser of the two, so a short -file quoting a license in full still matches — the bound it sets is on padding around a -text, not on a document that is mostly one. A file that genuinely carries two *distinct* -licenses is reported as carrying both; where one is a near-duplicate of the other, only the -match survives, and a disjunctive grant is not expressible here at all. - -Every grant a component ships is accounted for in its conclusion. A text that matches -nothing is not guessed at: it travels in the document as the component's ``LicenseRef-``, -which says "these are the terms that shipped" without claiming to know which license they -are, AND-ed with what its other texts are identified as — an MIT ``LICENSE`` beside an -unmatched EULA concludes ``LicenseRef- AND MIT``. A text matching identifiers -that share one reference text names the one the others extend by a suffix — ``MPL-2.0`` -beside ``MPL-2.0-no-copyleft-exception``, ``OFL-1.1`` beside its ``-RFN`` variants — since -that is SPDX's name for the unmodified text. Where there is none, as for ``GPL-3.0-only`` -and ``GPL-3.0-or-later``, the choice is the licensor's, stated in a notice and not in the -text, so the text is carried too. So is a text its identifiers account for less than four -fifths of, its copyright notices aside, beside the identifiers it names: the rest may be -terms no identifier names, as in fmt's MIT with its exception paragraph. The ``LicenseRef-`` carries exactly the texts -carried. Those components are listed in the report under *License evidence gaps*. - -A component whose only texts are a REUSE ``LICENSES/`` pool ships them, and that meets -the obligation below, but a pool says what each file is under, not what the component is. -Nothing is concluded for it, so its licence is ``NOASSERTION``, and it is listed under -*License evidence gaps* too — as is a component whose grants name nothing the grammar -accepts. - -A component with no licence text the collector reads blocks publication, because packaging -the text is the obligation the wheel has to meet. For a fetched component that is no licence -file at its root and, for a fetched tree, none in a ``LICENSES/`` pool and no licence -section in its README; a licence file in a subdirectory covers only that part, and terms -only its REUSE info states do not count. For material checked in here it is any one file -whose REUSE info names no licence, names one with no current SPDX reference text (a -``LicenseRef-`` among them, even beside one that has a text), states an expression that -does not parse, or names an id that is neither SPDX-listed nor a ``LicenseRef-`` or -``DocumentRef-``; an exception needs only to be listed as an exception. The build names -each such file and why, and another file of the same holder does not lend it its terms. The -remedy is to record upstream's terms as an ``SPDX-License-Identifier`` in the header, a -``.license`` sidecar or a ``REUSE.toml`` annotation. A tag that is not one — quoted in a -string, say, or prose that mentions the tag — goes between ``REUSE-IgnoreStart`` and -``REUSE-IgnoreEnd``, or is set aside by a ``.license`` sidecar or a ``REUSE.toml`` -annotation with ``precedence = "override"``. Where a checked-in third-party file's REUSE -info declares an identifier, the reference text for it ships, labelled in the document as a -substitution rather than as the component's own words — it conveys the terms and names no -holder, so the file's copyright notice travels with it. - -A shared library that none of those indexes explains did not come from this build, so the -last thing asked is the build machine itself: resolve the library's SONAME, and take its -license from the supplying package's copyright file, or from a license file beside the -library for toolchains that install outside the package manager. Resolving the SONAME is -what licenses the claim — a library the machine cannot find is reported as unexplained, -not attributed to something nobody located. The package's ``pkg:deb`` purl is namespaced -by the archive that publishes the installed version — the ``Origin`` apt records for it, -``ubuntu`` for Ubuntu's, ``nvidia`` for NVIDIA's CUDA repository. Where no archive lists -the installed version — superseded, typically, but equally a local rebuild — the archives -listing its other versions decide, and the build evidence says so. ``pkg:deb`` requires -that vendor, so where none can be named — no archive lists the package, as for a local -``.deb``; the deciding archives disagree, ignoring case, or name no ``Origin``; or the -``Origin`` is a URL or a host name, which a purl namespace cannot be — the package is -``pkg:generic/@`` instead. The build machine's own distribution is not -consulted. No repair tool's directory name is consulted, so a library the build produced -is attributed to the build wherever it ends up. Its copyright notice is read only from -the ``Copyright:`` fields of a machine-readable Debian copyright file, leaving out -stanzas that cover only the Debian packaging; any other text — an EULA, a free-form -copyright file — quotes the notices of what it bundles, so none is asserted, and the -packaged text still carries them all. - -Static code leaves no ``DT_NEEDED`` entry behind, so the link line is where it is read — -every fragment CMake records for it, ``-Wl,`` and ``-Xlinker`` words included, with the -values of options known to take one set aside. A path on it is judged by its content, not -its name: an archive or an object file is static code, a shared library is left to -``DT_NEEDED``, and a GNU ``ld`` script — ``libc.so`` is one, naming ``libc_nonshared.a`` — -is followed to the files and ``-l`` names its ``INPUT`` and ``GROUP`` commands give, a -relative name looked up beside the script first, as ``ld`` does. A -``-l`` name is resolved the way ``ld`` resolves it: through the line's ``-L`` directories, -then the directories the compiler hands the linker — CMake's toolchains reply, or the -compiler's own ``-print-search-dirs`` for a language that reply does not cover — then the -linker's built-in ``SEARCH_DIR`` list from ``ld --verbose``, taking a shared library before -an archive unless ``-Bstatic`` is in force. A leading ``=`` in either names the sysroot — -the target's ``CMAKE_SYSROOT``, else the compiler's — as does an absolute name in a script -inside it. ``--push-state`` and ``--pop-state`` are -honoured, and the driver's ``-static`` governs the whole line. The lines read are the -member's own and those of shared libraries it depends on; an executable or module it -merely waits for, such as a code generator, links nothing into it, and neither does -anything reached only through one. - -What is found is then attributed: - -- an archive or object a target of this build produced is that target's contribution; -- one inside a fetched component's tree is that component's; -- one vcpkg installed belongs to the port whose install record - (``vcpkg/info/__.list``) names it. The port is identified by the - upstream source its own ``share//vcpkg.spdx.json`` says it built — a ``pkg:github`` - purl where its one source is on GitHub, ``pkg:generic`` with no supplier where it is not - or where several sources name no single upstream, since purl has no vcpkg type — at the - upstream version, without vcpkg's ``#`` recipe revision. Its licence is read from - ``share//copyright``; -- one tracked in this repository belongs to the holder its REUSE info names, and is this - project's own where that is this project; -- one from the build machine is asked about the same way as a shared library: the package - that owns it and that package's copyright file, or, for an archive under a CUDA toolkit, - the license file the toolkit ships; -- an archive with no members links no code — glibc 2.34 left ``librt.a``, ``libpthread.a`` - and ``libdl.a`` as such stubs — and is recorded in the build evidence rather than as a - component. - -Each component found this way is recorded with ``STATIC_LINK``. Anything else stops the -build: a name that resolves to nothing, a fragment that does not parse, a path that is none -of those kinds — which is also what an unknown option's value looks like — and an archive -inside the build tree that no target produced and no install record names, which is where -ExternalProject leaves its output. Not inventoried: what the compiler driver adds without -naming it on the link line, ``libgcc`` and the C runtime's start files among it; and a -vcpkg port's headers read through an include directory other ports also install into. -Every port shares one include root, so an include directory credits a port only where each -file vcpkg installed under it is that port's; headers read from the shared root are -credited only through a linked archive of the same port. - -Running the collector locally ------------------------------ - -Against a configured build tree: - -.. code-block:: bash - - cmake -B build -DCMAKE_BUILD_TYPE=Release - cmake --build build - cmake --install build - - # Configuring fetched the SPDX reference texts into - # build/_deps/license-list-data-src/json, which is where `build` looks - # unless --license-data says otherwise. - uv run --project scripts/sbom isaaccapture-sbom build \ - --build-dir build --wheel install/wheels/.whl --out-dir sbom - uv run --project scripts/sbom isaaccapture-sbom check --wheel install/wheels/.whl \ - --manifest sbom/manifest.json --build-evidence sbom/.build-evidence.json - -``build`` rewrites the wheel in place, so run it on a repaired wheel and do not touch the -wheel afterwards. Set ``SOURCE_DATE_EPOCH`` to make the result reproducible. - -``--project`` points ``uv`` at the collector, not at this repository: it resolves the -collector's own pins and runs it from wherever you are. The checkout it inventories is the -one you are standing in, or ``--repo-root``. - -Maintaining it --------------- - -Adding a dependency takes no SBOM change: declare it the way you already would — a -``FetchContent_Declare``, a tarball under ``deps/`` — and it appears in the next wheel's -inventory with its identity, its license and the route it took to get there. - -These do need attention: +``check`` confirms the SBOM accounts for every file, its digests match the wheel and its +``RECORD``, each license text it cites is packaged where it says, and every +``LicenseRef-`` carries its text; add ``--build-evidence`` or ``--manifest`` to check +against the release evidence too. -- **A component with no license text the collector reads stops the build.** The fix is to - obtain the terms, not to record an assumption; a fetched component's REUSE info, source - headers included, is not read for them. For a file checked in here, obtain the terms, - then record them as REUSE licence info naming current SPDX-listed identifiers; a stray - tag that does not parse or names no such id is set aside instead. Both remedies are - under `How licenses are determined`_. -- **A new member that the build cannot explain stops the build.** That means something - reached the wheel by a route the collector cannot see; teach it that route rather than - excluding the file. -- **Static code on a link line that the collector cannot trace stops the build.** Link - what it can trace — a target of this build, a fetched dependency, a system package or a - known toolkit — or teach it the route that input took, such as an ExternalProject install - into the build tree. +The SBOM describes a wheel; it is not a signature. Integrity is the wheel's ``RECORD``, +which installers check. These releases carry no +`PEP 740 `_ attestations. -Moving to a newer SPDX license list is a dependency bump like any other: change -``SPDX_LICENSE_DATA_COMMIT`` in :code-file:`deps/third_party/CMakeLists.txt`, with its release -comment. There is no separate list of identifiers to extend — the whole SPDX list comes -with the checkout. +To read the inventory, open ``.dist-info/sboms/*.spdx.json`` with any SPDX 2.3 tool. A +file's ``SPDXID`` changes only when its path does, so two releases can be diffed. diff --git a/scripts/sbom/README.md b/scripts/sbom/README.md new file mode 100644 index 0000000000..16de4ddc2d --- /dev/null +++ b/scripts/sbom/README.md @@ -0,0 +1,387 @@ + + +# Wheel SBOM collector + +`isaaccapture-sbom` derives a wheel's contents SBOM and license evidence from the build +that produced it, packages both into the wheel, and checks the result. What a wheel +carries and how to verify one are on the user page, +[Wheel contents SBOM and license evidence](https://nvidia.github.io/IsaacCapture/main/references/sbom.html) +(source: [`docs/source/references/sbom.rst`](../../docs/source/references/sbom.rst)). +This file is how it works. + +Where to look: running or testing it, [Running it](#running-it); a member the build +cannot explain, [How the inventory is worked out](#how-the-inventory-is-worked-out); a +wrong or missing license, [How licenses are determined](#how-licenses-are-determined); a +library from the build machine or static code, +[Libraries from the build machine](#libraries-from-the-build-machine) and +[Static code on the link line](#static-code-on-the-link-line); bumping the SPDX list, +[Reference texts](#reference-texts); what stops the build and how to fix it, +[What stops the Release build](#what-stops-the-release-build). + +## Running it + +The collector is a project of its own; its dependencies are pinned in +[`pyproject.toml`](pyproject.toml). `reuse` reads files through libmagic, so `build` +needs it (`libmagic1` on Ubuntu); `check` does not. + +Build and install a Release wheel as in +[Building from source](../../docs/source/development/build_from_source/index.rst), then: + +```bash +# `build` reads the SPDX reference texts configuring fetched into +# build/_deps/license-list-data-src/json, unless --license-data says otherwise. +uv run --project scripts/sbom isaaccapture-sbom build \ + --build-dir build --wheel install/wheels/.whl --out-dir sbom +uv run --project scripts/sbom isaaccapture-sbom check --wheel install/wheels/.whl \ + --manifest sbom/manifest.json --build-evidence sbom/.build-evidence.json +``` + +- `build` rewrites the wheel in place, so run it on a repaired wheel and do not touch + the wheel afterwards. Set `SOURCE_DATE_EPOCH` to make the result reproducible. +- `auditwheel` writes its own CycloneDX document into `.dist-info/sboms/`; the collector + finds its document there by name and leaves the rest alone. +- `--project` makes `uv` resolve the collector's pins and run it from anywhere. It + inventories the checkout you are in, or `--repo-root`. + +### What `check` verifies + +The SBOM's per-file digests show that the description belongs to the wheel in front of +you and agrees with its `RECORD`; they are not a second root of trust. The document +lists every member except `RECORD` and itself, since neither can carry its own digest; +`check` reads both against the archive directly. With just a wheel it confirms that + +- the SBOM covers every file in the wheel except `RECORD` and the SBOM itself, +- each digest the SBOM records matches the bytes present, and agrees with `RECORD`, +- `RECORD` itself agrees with the archive, so the wheel is still installable, +- the packaged notices are there, and every `LicenseRef-` has its text embedded. + +`--build-evidence` also checks that the build accounts for every member; `--manifest` +checks the wheel against the digest bound at publication and the sidecars beside it. +`check-set` runs the whole thing over every wheel in a release, and fails a wheel in +`--wheel-dir` that the manifest does not list. Any command reading a manifest refuses +one that lists no wheel. CI validates each published document with `spdx-tools` before +release. + +### Tests + +From the repository root: + +```bash +uv run --project scripts/sbom --extra dev pytest scripts/sbom/tests +``` + +Every test reads the SPDX list from `ISAACCAPTURE_SBOM_LICENSE_DATA`, else +`build/_deps/license-list-data-src/json`, and skips when neither holds it. The file API +tests also read `ISAACCAPTURE_SBOM_BUILD_DIR`, else `build/`, and skip without a +configured tree. With `BUILD_TESTING`, [`CMakeLists.txt`](CMakeLists.txt) registers each +test file with CTest as `sbom_`, pointing both variables at the current build. + +## How the inventory is worked out + +Every inventory entry is derived from the build, from these sources: + +**`build/_deps/*-src`** — FetchContent trees. + +- A tree that is a git repository of its own is asked for its commit and remote, and + from the remote the supplier, homepage and purl. Git is asked of no other tree, since + it would answer for whichever checkout encloses the build directory, or for whatever + `GIT_DIR` names; `GIT_DIR` is therefore cleared. +- Any other tree is identified by its declaration — `GIT_REPOSITORY` and `GIT_TAG`, or + `URL` and `URL_HASH` — from the stamps ExternalProject writes beside the populate + sub-build, or under `CMakeFiles/fc-stamp` once policy `CMP0168` is `NEW`, and from the + clone script the git stamp names. A stamp outlives a change of method, so where both + are present the tree decides: a git clone is a checkout of its own. +- The version is the declared tag, or the ref a GitHub URL names, verbatim, so a tag + gives one version whether declared by tag or as a GitHub archive. Any other archive's + version is read from its name, and its supplier is not asserted: the host serving a + download is not who supplies it. +- A download location is stated only for an `http`, `https`, `ftp` or `ftps` archive + URL, or a git remote reached over `http`, `https`, `ssh` or `git`, and never with its + credentials, query or fragment. + +**vcpkg's install records** — the `vcpkg/info` lists under `VCPKG_INSTALLED_DIR` (by +default the build tree's `vcpkg_installed`): which port installed each file. + +**CMake's file API codemodel** — the build graph as CMake describes it: which sources +were compiled into which binary, which targets it depends on, what its link line names +and, through include paths, which header-only dependencies it was compiled against. +Followed transitively, so a library linked into a library linked into the wheel is still +reported. The top-level `CMakeLists.txt` requests it with `cmake_file_api()`; the +project builds without it, but a wheel with an SBOM needs it, hence the CMake version in +[Prerequisites](../../docs/source/development/build_from_source/index.rst#prerequisites). +Do not read the generator's own output instead: Ninja writes no `link.txt`. + +**The compiler's dependency output** — which headers each object read: the per-object +depfiles and `compiler_depend.make` under the Makefile generators, `ninja -t deps` under +Ninja. A Ninja build is collected where `ninja` is on `PATH`; the collector says so once +when it is not. A target whose records miss a source it compiles falls back to its +listed sources and include directories. + +**Archives in the repository** — every member of every tarball is hashed; a wheel member +that matches one is recorded as unpacked from that archive. + +**The working tree and the build tree** — what remains is matched against the project's +own sources and its generated output. + +**The build machine** — for a shared library none of the above explains; see +[Libraries from the build machine](#libraries-from-the-build-machine). + +A member none of these explains stops the build +([below](#what-stops-the-release-build)). Adding a dependency needs no change here: +declare it as usual — a `FetchContent_Declare`, a tarball under `deps/` — and it appears +in the next wheel's inventory. + +Post-processing is recorded as such. `patchelf`, `auditwheel`'s RPATH rewrite and the +build's MJCF asset stripping leave bytes that no longer hash to their origin, so those +members are reported as *derived from* a named input rather than as an exact copy, with +the input's digest wherever the input was hashed: an archive member is, a file matched +only by its path is not. + +A file's `SPDXID` is its path in the distribution, reduced to the characters an SPDX +identifier allows, plus a short digest of the unreduced path, with the versioned +`.dist-info` and `.data` directories counted as fixed. + +### Libraries from the build machine + +A shared library none of the build's indexes explains did not come from this build, so +the build machine is asked: resolve its SONAME and take its license from the supplying +package's copyright file, or from a license file beside the library for toolchains that +install outside the package manager. A library the machine cannot find is reported as +unexplained, not attributed. + +- The `pkg:deb` purl is namespaced by the archive publishing the installed version: the + `Origin` apt records for it (`ubuntu` for Ubuntu's, `nvidia` for NVIDIA's CUDA + repository). Where no archive lists the installed version — superseded, or a local + rebuild — the archives listing its other versions decide, and the build evidence says + so. +- Where no vendor can be named — no archive lists the package (a local `.deb`); the + deciding archives disagree, ignoring case, or name no `Origin`; or the `Origin` is a + URL or host name, which a purl namespace cannot be — the package is + `pkg:generic/@`. The build machine's own distribution is not + consulted. +- No repair tool's directory name is consulted, so a library the build produced is + attributed to the build wherever it ends up. +- Copyright notices are read only from the `Copyright:` fields of a machine-readable + Debian copyright file, leaving out stanzas that cover only the Debian packaging. Any + other text (an EULA, a free-form copyright file) quotes the notices of what it + bundles, so none is asserted; the packaged text still carries them all. + +### Static code on the link line + +Static code leaves no `DT_NEEDED` entry, so the link line is read: every fragment CMake +records for it, `-Wl,` and `-Xlinker` words included, with the values of options known +to take one set aside. + +- A path is judged by its content, not its name: an archive or object file is static + code, a shared library is left to `DT_NEEDED`, and a GNU `ld` script (`libc.so`, + naming `libc_nonshared.a`) is followed to the files and `-l` names its `INPUT` and + `GROUP` commands give, a relative name looked up beside the script first, as `ld` + does. +- A `-l` name is resolved as `ld` resolves it: the line's `-L` directories, then the + directories the compiler hands the linker (CMake's toolchains reply, or the compiler's + own `-print-search-dirs` for a language that reply does not cover), then + `ld --verbose`'s built-in `SEARCH_DIR` list, taking a shared library before an archive + unless `-Bstatic` is in force. A leading `=` in either names the sysroot (the target's + `CMAKE_SYSROOT`, else the compiler's), as does an absolute name in a script inside it. + `--push-state` and `--pop-state` are honoured, and the driver's `-static` governs the + whole line. +- The lines read are the member's own and those of shared libraries it depends on. An + executable or module it merely waits for, such as a code generator, links nothing into + it, nor does anything reached only through one. + +What is found is attributed: + +- an archive or object a target of this build produced is that target's contribution; +- one inside a fetched component's tree is that component's; +- one vcpkg installed belongs to the port whose install record + (`vcpkg/info/__.list`) names it. The port is identified by the + upstream source its `share//vcpkg.spdx.json` says it built — a `pkg:github` purl + where its one source is on GitHub, `pkg:generic` with no supplier where it is not or + where several sources name no single upstream, since purl has no vcpkg type — at the + upstream version, without vcpkg's `#` recipe revision. Its license is read from + `share//copyright`; +- one tracked in this repository belongs to the holder its REUSE info names, and is this + project's own where that is this project; +- one from the build machine is asked about as a shared library is: the owning package + and its copyright file or, for an archive under a CUDA toolkit, the license file the + toolkit ships; +- an archive with no members links no code — glibc 2.34 left `librt.a`, `libpthread.a` + and `libdl.a` as such stubs — and is recorded in the build evidence rather than as a + component. + +Each component found this way is recorded with `STATIC_LINK`; anything else stops the +build ([below](#what-stops-the-release-build)). + +Not inventoried: what the compiler driver adds without naming it on the link line +(`libgcc` and the C runtime's start files among it), and a vcpkg port's headers read +through an include directory other ports also install into. Every port shares one +include root, so an include directory credits a port only where each file vcpkg +installed under it is that port's; headers read from the shared root are credited only +through a linked archive of the same port. + +## How licenses are determined + +Licenses are never inferred from this project's license, and never read from a list +maintained by hand. + +### Where terms are read from + +- **Files in this repository** — from their REUSE info + ([License headers](../../docs/source/development/license.rst), + [REUSE specification](https://reuse.software/spec/)). The pre-commit hook lints it for + source files; the collector reads it for every file REUSE covers, whatever its type, + and for every source here the build compiled or included. A file identified as a third + party's (below) is held to the per-file gate even where the hook does not lint it; a + file naming no holder at all counts as this project's own. +- **Fetched components** — from the license and notice files the component ships: + `LICENSE`, `COPYING`, `BSD-LICENSE`, a `LICENSES/` pool or, where none of those at its + root is a grant, the license section of its README. Each text is matched against the + SPDX reference texts. Its source headers, `.license` sidecars and `REUSE.toml` are not + read. +- **Archives in the repository** — only the license files at the archive's root. + +### Reference texts + +[`deps/third_party/CMakeLists.txt`](../../deps/third_party/CMakeLists.txt) fetches +`spdx/license-list-data` at a pinned commit into `_deps/license-list-data-src`, like any +other dependency, so the collector reports it as a component fetched but shipping +nothing. Each report records the commit its identifications were made against. + +Every current identifier is available to match against. Naming one in the document is a +separate question: the document states the license list it was built against, and +`license-expression` — which consumers validate with and `pyproject.toml` pins — ships +its own, older copy. An id the fetched list knows and that grammar does not cannot be +published as an id, so the text it was read from travels as a `LicenseRef-`. A REUSE +declaration naming one travels whole, since an `OR` or a `WITH` cannot be kept in part. +The component is listed under *License evidence gaps* with that reason. + +To move to a newer list, change `SPDX_LICENSE_DATA_COMMIT` in +`deps/third_party/CMakeLists.txt`, with its release comment, and the entry in +[`deps/README.md`](../../deps/README.md#spdx-license-list-data). There is no separate +list of identifiers to extend. + +### What ships + +- **A fetched component** ships its own license file verbatim, copyright holders and + all. Do not substitute a reference text: it would drop the notice — for MIT and the + BSD licenses the difference is the line naming who licensed it to you — and replace + the component's actual grant with our reading of it. +- **Third-party material checked in here** is identified by the copyright holder it is + declared under: a `.license` sidecar or `REUSE.toml` annotation where REUSE applies + one, and the file's own first copyright line unless REUSE applies those in its place. + Its terms are the identifiers REUSE gives each of its files, under REUSE's own + precedence. The reference text for each license it names ships, labelled in the + document as a substitution, since a reader cannot tell a template from an author's + notice; the file's copyright notice travels with it. The expression keeps its + structure, so a choice stays a choice; an exception's text ships too where the + expression travels as a `LicenseRef-`. +- **Terms stated for part of a component** — a vendored asset set or bundled library + with its own `LICENSE` and holder — are read and packaged only where something under + that directory reached the wheel: a file copied or derived from it (including a copy + the build made into the component's own build tree), a source compiled from it, or a + header the compiler's dependency output says it read. Where a target compiles + something its dependency output does not account for, the sources it lists count + instead, and so does any include directory above or below that directory. A linked + archive of the component keeps all of them, since what it holds is unknown. A + `.license` sidecar covers only the file it annotates. The build evidence says which of + those kept each text. +- **A bundled library compiled in** ships its texts and holders, each listed as terms + stated for part of the component; the component's concluded license stays its own + grant. +- **The distribution's own declared license** is an obligation like any other. Where the + wheel ships no text for it, the reference text for each identifier is packaged from + the pinned list and declared under PEP 639. +- **The notices file** opens with the distribution's own license and copyright holder, + read from the packaging metadata where no file it ships states one. Otherwise a wheel + shipping only metadata would carry a template naming nobody. +- **Never a file from `LICENSES/`** as a third party's terms: that pool licenses *this* + project and carries its notices. Substituted texts come from the pinned list. Under + `.dist-info/licenses/LICENSES/` a wheel carries the project's own pool files where it + has them, and otherwise the reference text for the license it declares. + +`licenseDeclared` on the wheel is what the distribution says of itself; +`licenseConcluded` is what the document found in it. A wheel carrying a component whose +terms the build could not name states that component's `LicenseRef-` alongside the +declaration. Where nothing at all is concluded for a component, the wheel's conclusion +is `NOASSERTION`. + +### Matching + +A match needs a canonical license text to be almost entirely present *and* to account +for a real share of the file, so a license reproduced in an appendix of a long agreement +is not that agreement's terms. The second threshold is the looser: it bounds padding +around a text, so a short file quoting a license in full still matches. A file carrying +two *distinct* licenses is reported as carrying both; where one is a near-duplicate of +the other, only the match survives. A disjunctive grant is not expressible. + +Every grant a component ships is accounted for in its conclusion: + +- A text that matches nothing travels as the component's `LicenseRef-`, AND-ed with what + its other texts are identified as: an MIT `LICENSE` beside an unmatched EULA concludes + `LicenseRef- AND MIT`. +- A text matching identifiers that share one reference text names the one the others + extend by a suffix — `MPL-2.0` beside `MPL-2.0-no-copyleft-exception`, `OFL-1.1` + beside its `-RFN` variants — SPDX's name for the unmodified text. Where there is none, + as for `GPL-3.0-only` and `GPL-3.0-or-later`, the choice is the licensor's, stated in + a notice rather than the text, so the text is carried too, and the evidence lists it + as matching either, not both. +- A text its identifiers account for less than four fifths of, copyright notices aside, + is carried too, beside the identifiers it names: the rest may be terms no identifier + names, as in fmt's MIT with its exception paragraph. + +The `LicenseRef-` carries exactly the texts carried, and those components are listed +under *License evidence gaps*. So is a component whose only texts are a REUSE +`LICENSES/` pool: it ships them, which meets the obligation, but a pool says what each +file is under, not what the component is, so its license is `NOASSERTION`. So is a +component whose grants name nothing the grammar accepts. + +## What stops the Release build + +The build stops rather than ship a gap, and its error names the files or components +involved. None of these is fixed by excluding a file or recording an assumed license. +Other errors — an unconfigured build tree, no file API codemodel, an unreadable +archive — are not gaps, and name their own cause. + +**A wheel member the build cannot explain** — none of the +[inventory's sources](#how-the-inventory-is-worked-out) accounts for it, including a +shared library the build machine cannot resolve. It reached the wheel by a route the +collector cannot see: teach the collector that route rather than excluding the file. + +**Static code on a link line the collector cannot trace** — a name that resolves to +nothing, a fragment that does not parse, a path that is none of the +[kinds above](#static-code-on-the-link-line) (which is also what an unknown option's +value looks like), or an archive inside the build tree that no target produced and no +install record names, which is where ExternalProject leaves its output. Link something +the collector can trace — a target of this build, a fetched dependency, a system package +or a known toolkit — or teach it the route that input took. + +**A redistributed component with no license text** — packaging the text is the +obligation. For a fetched component, that is none of the texts +[Where terms are read from](#where-terms-are-read-from) lists; a license file in a +subdirectory covers only that part. For any other component it is the source named above +for its kind: an archive's root license files, a vcpkg port's `share//copyright`, +a build-machine library's package copyright file or the license file beside it. Obtain +the terms; the component's REUSE info and source headers do not count. + +**A declared license with no text shipped** — the wheel's `License-Expression` does not +parse, or names an identifier the pinned list cannot supply a text for, such as a +`LicenseRef-`, and the wheel packages none. A text counts only as `.txt` +outside `third-party/`: add it to `license-files` in `pyproject.toml` under that name, or +drop the declaration. + +**A checked-in third-party file whose REUSE info fails the per-file gate** — the file's +REUSE info names no license; names one with no current SPDX reference text (a +`LicenseRef-` among them, even beside one that has a text); states an expression that +does not parse; or names an id that is neither SPDX-listed nor a `LicenseRef-` or +`DocumentRef-`. An exception needs only to be listed as an exception. The build names +each such file and why; another file of the same holder does not lend it its terms. + +- For real upstream terms, record them as REUSE info naming current SPDX-listed + identifiers ([License headers](../../docs/source/development/license.rst), + [REUSE specification](https://reuse.software/spec/)). +- A tag that is not a real declaration — quoted in a string, or mentioned in prose — + goes between `REUSE-IgnoreStart` and `REUSE-IgnoreEnd`, or is set aside by a + `.license` sidecar or a `REUSE.toml` annotation with `precedence = "override"`. diff --git a/scripts/sbom/isaaccapture_sbom/build.py b/scripts/sbom/isaaccapture_sbom/build.py index feb6fa07be..ae90c80e6f 100644 --- a/scripts/sbom/isaaccapture_sbom/build.py +++ b/scripts/sbom/isaaccapture_sbom/build.py @@ -78,6 +78,10 @@ class BuildError(Exception): """The wheel cannot be published with the evidence available.""" +# Each evidence-gap error ends with this; the README section holds the remedies. +_REMEDIES = "\nSee 'What stops the Release build' in scripts/sbom/README.md." + + def _entry(name: str, data: bytes) -> Entry: return Entry( name=name, @@ -275,7 +279,8 @@ def build( "these wheel members could not be traced to anything this build " "produced or fetched:\n " + "\n ".join(inventory.unattributed) - + "\nThey reached the wheel by a route the collector cannot see." + + "\nThey reached the wheel by a route the collector cannot see; teach " + "the collector that route rather than excluding the file." + _REMEDIES ) if inventory.untraced: @@ -286,7 +291,7 @@ def build( + "\nLink only what the collector can trace -- a target of this build, a " "fetched dependency's tree, a system package or a known toolkit -- or " "teach it the route this input took. ExternalProject installs into the " - "build tree by a route it does not know." + "build tree by a route it does not know." + _REMEDIES ) unlicensed = sorted( @@ -300,8 +305,8 @@ def build( "the per-file gate:\n " + "\n ".join(unlicensed) + "\nRecord upstream's terms as REUSE info: a header tag, a .license " - "sidecar or a REUSE.toml annotation. For a stray tag, see 'How licenses " - "are determined' in docs/source/references/sbom.rst." + "sidecar or a REUSE.toml annotation. A stray tag is set aside instead." + + _REMEDIES ) missing = [ @@ -314,7 +319,7 @@ def build( "no license text was found in the files of redistributed component(s): " + ", ".join(missing) + "\nThe component ships no LICENSE/COPYING file and states no grant in " - "its README; obtain the terms before publishing it." + "its README; obtain the terms before publishing it." + _REMEDIES ) # Everything the wheel gains, computed before a byte is written so the @@ -373,7 +378,8 @@ def build( f"{wheel_path.name} declares {declared} and packages no text for " + ", ".join(unmet) + ".\nA distribution has to ship the terms it declares, the same as " - "anything it redistributes; obtain the text before publishing it." + "anything it redistributes: add the text to license-files in " + "pyproject.toml as .txt, or drop the declaration." + _REMEDIES ) patched_metadata = wheelfile.add_license_files(metadata_raw, sorted(packaged_paths)) diff --git a/scripts/sbom/isaaccapture_sbom/discovery.py b/scripts/sbom/isaaccapture_sbom/discovery.py index b983d6b35b..8ede6ece7a 100644 --- a/scripts/sbom/isaaccapture_sbom/discovery.py +++ b/scripts/sbom/isaaccapture_sbom/discovery.py @@ -324,7 +324,10 @@ class _Declaration: def _stamps(deps_dir: Path) -> dict[str, dict[str, dict[str, str]]]: - """Each tree's FetchContent stamps, by method; sbom.rst says where from.""" + """Each tree's FetchContent stamps, by method. + + scripts/sbom/README.md says where from. + """ # Under the populate sub-build, or, with CMP0168 NEW, the build's own CMakeFiles. roots = ( (deps_dir, "*-subbuild/*-populate-prefix/src/*-populate-stamp"), diff --git a/scripts/sbom/pyproject.toml b/scripts/sbom/pyproject.toml index b4f755aa73..31c61b2db7 100644 --- a/scripts/sbom/pyproject.toml +++ b/scripts/sbom/pyproject.toml @@ -7,7 +7,7 @@ # uv run --project scripts/sbom isaaccapture-sbom --help # uv run --project scripts/sbom --extra dev pytest scripts/sbom/tests # -# See docs: references/sbom. +# How it works: README.md here. What a wheel carries: docs/source/references/sbom.rst. [project] name = "isaaccapture-sbom" From 05fa5932e6bd3bea7a148833e2e77f94918f71ae Mon Sep 17 00:00:00 2001 From: Andrei Aristarkhov Date: Tue, 20 Oct 2026 17:20:52 -0700 Subject: [PATCH 095/101] ci(sbom): the collector's own tests gate SBOM generation The collector's tests ran inside the general ctest step, ahead of SBOM generation only by accident of step order. They now carry the ctest label `sbom` and run as their own Release-only step, "Test the SBOM collector", straight after the general tests (which exclude the label), so a Release leg packages SBOMs only once they have passed. The gate cannot pass on nothing: `--no-tests=error` fails it when no test carries the label, and a path CTest names for the SPDX license data or the build tree that holds nothing now fails the tests instead of skipping them. Run outside CTest with nothing configured, they still skip. Signed-off-by: Andrei Aristarkhov --- .github/workflows/build-ubuntu.yml | 7 +++- scripts/sbom/CMakeLists.txt | 1 + scripts/sbom/README.md | 9 ++-- scripts/sbom/tests/conftest.py | 67 +++++++++++++++++------------- 4 files changed, 49 insertions(+), 35 deletions(-) diff --git a/.github/workflows/build-ubuntu.yml b/.github/workflows/build-ubuntu.yml index 2ab293320f..dc6db73839 100644 --- a/.github/workflows/build-ubuntu.yml +++ b/.github/workflows/build-ubuntu.yml @@ -174,7 +174,12 @@ jobs: run: ccache --show-stats - name: Run All Tests - run: ctest --test-dir "${BUILD_DIR}" --output-on-failure --parallel + run: ctest --test-dir "${BUILD_DIR}" -LE '^sbom$' --output-on-failure --parallel + + # The collector's own tests gate SBOM generation, which only Release legs do. + - name: Test the SBOM collector (Release only) + if: matrix.build_type == 'Release' + run: ctest --test-dir "${BUILD_DIR}" -L '^sbom$' --no-tests=error --output-on-failure --parallel - name: Generate coverage report if: ${{ matrix.build_type == 'Debug' && matrix.python_version == '3.11' && matrix.arch == 'x64' }} diff --git a/scripts/sbom/CMakeLists.txt b/scripts/sbom/CMakeLists.txt index c98b5dbcc9..3c5cfc60a4 100644 --- a/scripts/sbom/CMakeLists.txt +++ b/scripts/sbom/CMakeLists.txt @@ -24,5 +24,6 @@ foreach(test_file ${TEST_FILES}) # a codemodel CMake actually wrote. set_tests_properties("sbom_${test_name}" PROPERTIES ENVIRONMENT "ISAACCAPTURE_SBOM_LICENSE_DATA=${SPDX_LICENSE_DATA_JSON_DIR};ISAACCAPTURE_SBOM_BUILD_DIR=${CMAKE_BINARY_DIR}" + LABELS "sbom" ) endforeach() diff --git a/scripts/sbom/README.md b/scripts/sbom/README.md index 16de4ddc2d..bbfe826c91 100644 --- a/scripts/sbom/README.md +++ b/scripts/sbom/README.md @@ -74,10 +74,11 @@ uv run --project scripts/sbom --extra dev pytest scripts/sbom/tests ``` Every test reads the SPDX list from `ISAACCAPTURE_SBOM_LICENSE_DATA`, else -`build/_deps/license-list-data-src/json`, and skips when neither holds it. The file API -tests also read `ISAACCAPTURE_SBOM_BUILD_DIR`, else `build/`, and skip without a -configured tree. With `BUILD_TESTING`, [`CMakeLists.txt`](CMakeLists.txt) registers each -test file with CTest as `sbom_`, pointing both variables at the current build. +`build/_deps/license-list-data-src/json`; the file API tests also read +`ISAACCAPTURE_SBOM_BUILD_DIR`, else `build/`. A default that holds nothing skips them; a +variable that names a path holding nothing fails them. With `BUILD_TESTING`, [`CMakeLists.txt`](CMakeLists.txt) registers each +test file with CTest as `sbom_` under the label `sbom`, pointing both variables at +the current build. ## How the inventory is worked out diff --git a/scripts/sbom/tests/conftest.py b/scripts/sbom/tests/conftest.py index 9574772eff..8b502e0797 100644 --- a/scripts/sbom/tests/conftest.py +++ b/scripts/sbom/tests/conftest.py @@ -24,35 +24,46 @@ def repo_root() -> Path | None: return None -def _candidates(env: str, *in_checkout: str): - """The environment override, then the checkout's default, asked lazily.""" +def _located(env: str, usable, missing: str, *in_checkout: str) -> Path: + """The path `env` names, else the checkout's default, if `usable` accepts it. + + A path the environment names is one CTest promised, so its absence fails + every test; without one, the tests skip, since nothing was promised. + """ if os.environ.get(env): - yield Path(os.environ[env]) + named = Path(os.environ[env]) + if usable(named): + return named + pytest.fail(f"{env}={named}: {missing}") root = repo_root() - if root is not None: - yield root.joinpath(*in_checkout) + if root is not None and usable(root.joinpath(*in_checkout)): + return root.joinpath(*in_checkout) + pytest.skip(f"{missing}; configure this project, or set {env}") + + +def _corpus_loads(path: Path) -> bool: + try: + licensing.load_corpus(path) + except licensing.CorpusError: + return False + return True + + +def _has_codemodel(path: Path) -> bool: + return any((path / ".cmake" / "api" / "v1" / "reply").glob("codemodel-v2-*.json")) @pytest.fixture(scope="session") def license_data(): - """The SPDX license list deps/third_party fetched into this build's _deps. - - CTest points the environment variable at it. Outside that, fall back to a - default build directory; if neither exists the license tests skip rather - than run against data that is not what the build pinned. - """ - for candidate in _candidates( - LICENSE_DATA_ENV, "build", "_deps", "license-list-data-src", "json" - ): - try: - licensing.load_corpus(candidate) - return candidate - except licensing.CorpusError: - continue - - pytest.skip( - "SPDX license list data not found; configure and build so deps/third_party " - "materializes it, or set " + LICENSE_DATA_ENV + """The SPDX license list deps/third_party fetched into this build's _deps.""" + return _located( + LICENSE_DATA_ENV, + _corpus_loads, + "no SPDX license list data there", + "build", + "_deps", + "license-list-data-src", + "json", ) @@ -94,13 +105,9 @@ def live_build_dir(): the anchor to what it actually emits, so a change to the codemodel's shape fails here instead of passing everywhere. """ - for candidate in _candidates(BUILD_DIR_ENV, "build"): - if sorted( - (candidate / ".cmake" / "api" / "v1" / "reply").glob("codemodel-v2-*.json") - ): - return candidate - - pytest.skip("no configured build tree to read; configure this project first") + return _located( + BUILD_DIR_ENV, _has_codemodel, "no CMake file API codemodel there", "build" + ) @pytest.fixture From cc1a78ff650a24ac50bc9e79f5675c325eac4283 Mon Sep 17 00:00:00 2001 From: Andrei Aristarkhov Date: Wed, 21 Oct 2026 10:13:47 -0700 Subject: [PATCH 096/101] build(sbom): run the collector's tests with uv, outside the product's CMake The collector is a uv project of its own, and so are its tests: the product's CMake no longer registers them and CTest no longer runs them. Ubuntu's Release legs run them with uv directly, before any SBOM is generated, so a failure still stops generation. The tests take one input, the build tree ISAACCAPTURE_SBOM_BUILD_DIR names, and find the SPDX list in it the way the collector does; ISAACCAPTURE_SBOM_LICENSE_DATA is gone. A named tree that lacks either input fails the tests rather than skipping them. Signed-off-by: Andrei Aristarkhov --- .github/workflows/build-ubuntu.yml | 9 ++++++-- AGENTS.md | 2 ++ CMakeLists.txt | 1 - deps/third_party/CMakeLists.txt | 5 +---- scripts/sbom/CMakeLists.txt | 29 ------------------------- scripts/sbom/README.md | 10 ++++----- scripts/sbom/tests/conftest.py | 35 +++++++++++------------------- 7 files changed, 27 insertions(+), 64 deletions(-) delete mode 100644 scripts/sbom/CMakeLists.txt diff --git a/.github/workflows/build-ubuntu.yml b/.github/workflows/build-ubuntu.yml index dc6db73839..c998d0e028 100644 --- a/.github/workflows/build-ubuntu.yml +++ b/.github/workflows/build-ubuntu.yml @@ -174,12 +174,17 @@ jobs: run: ccache --show-stats - name: Run All Tests - run: ctest --test-dir "${BUILD_DIR}" -LE '^sbom$' --output-on-failure --parallel + run: ctest --test-dir "${BUILD_DIR}" --output-on-failure --parallel # The collector's own tests gate SBOM generation, which only Release legs do. + # Naming the build makes a missing input fail the tests, not skip them. - name: Test the SBOM collector (Release only) if: matrix.build_type == 'Release' - run: ctest --test-dir "${BUILD_DIR}" -L '^sbom$' --no-tests=error --output-on-failure --parallel + env: + ISAACCAPTURE_SBOM_BUILD_DIR: ${{ env.BUILD_DIR }} + run: | + uv run --project scripts/sbom --python ${{ matrix.python_version }} \ + --extra dev pytest scripts/sbom/tests - name: Generate coverage report if: ${{ matrix.build_type == 'Debug' && matrix.python_version == '3.11' && matrix.arch == 'x64' }} diff --git a/AGENTS.md b/AGENTS.md index b324e6b5c1..0113244048 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -292,6 +292,8 @@ build'). The collector is a project of its own: `uv run --project scripts/sbom isaaccapture-sbom ...`. Its dependency pins live in `scripts/sbom/pyproject.toml` and nowhere else — never restate them as `--with` flags in a workflow or a doc. +Its tests run the same way, through uv; do not register them with the product's +CMake or CTest. **Read [`scripts/sbom/README.md`](scripts/sbom/README.md) first** if you are changing anything under `scripts/sbom/`, changing how a dependency reaches a diff --git a/CMakeLists.txt b/CMakeLists.txt index 28211c5919..e666d0be98 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -175,7 +175,6 @@ endif() # Tests (pytest + Catch2/CTest). After production libs so link targets exist. if(BUILD_TESTING) add_subdirectory(tests) - add_subdirectory(scripts/sbom) endif() # Formatting enforcement (runs on Linux by default) diff --git a/deps/third_party/CMakeLists.txt b/deps/third_party/CMakeLists.txt index d83abf5bbd..15dc783a0a 100644 --- a/deps/third_party/CMakeLists.txt +++ b/deps/third_party/CMakeLists.txt @@ -309,11 +309,8 @@ FetchContent_Declare( ) FetchContent_MakeAvailable(license_list_data) -# What the SBOM collector reads, and what the sbom tests run against. -set(SPDX_LICENSE_DATA_JSON_DIR "${SPDX_LICENSE_DATA_SOURCE_DIR}/json" - CACHE INTERNAL "SPDX reference license texts for the SBOM collector") foreach(_spdx_index licenses.json exceptions.json) - if(NOT EXISTS "${SPDX_LICENSE_DATA_JSON_DIR}/${_spdx_index}") + if(NOT EXISTS "${SPDX_LICENSE_DATA_SOURCE_DIR}/json/${_spdx_index}") message(FATAL_ERROR "SPDX license list data is missing json/${_spdx_index} after fetch") endif() endforeach() diff --git a/scripts/sbom/CMakeLists.txt b/scripts/sbom/CMakeLists.txt deleted file mode 100644 index 3c5cfc60a4..0000000000 --- a/scripts/sbom/CMakeLists.txt +++ /dev/null @@ -1,29 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -# Registers the SBOM collector's tests with CTest; they need no built package. - -file(GLOB TEST_FILES - RELATIVE "${CMAKE_CURRENT_SOURCE_DIR}" - "${CMAKE_CURRENT_SOURCE_DIR}/tests/test_*.py" -) - -foreach(test_file ${TEST_FILES}) - get_filename_component(test_name "${test_file}" NAME_WE) - - add_test( - NAME "sbom_${test_name}" - COMMAND uv run --project "${CMAKE_CURRENT_SOURCE_DIR}" --python ${ISAAC_TELEOP_PYTHON_VERSION} - --extra dev pytest -v --tb=short "${test_file}" - WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}" - ) - - # The SPDX license list deps/third_party fetched for this build, so the tests - # read the same reference texts the collector will and never reach the - # network; and this build tree, so the file API reader is exercised against - # a codemodel CMake actually wrote. - set_tests_properties("sbom_${test_name}" PROPERTIES - ENVIRONMENT "ISAACCAPTURE_SBOM_LICENSE_DATA=${SPDX_LICENSE_DATA_JSON_DIR};ISAACCAPTURE_SBOM_BUILD_DIR=${CMAKE_BINARY_DIR}" - LABELS "sbom" - ) -endforeach() diff --git a/scripts/sbom/README.md b/scripts/sbom/README.md index bbfe826c91..cdb2782d5c 100644 --- a/scripts/sbom/README.md +++ b/scripts/sbom/README.md @@ -73,12 +73,10 @@ From the repository root: uv run --project scripts/sbom --extra dev pytest scripts/sbom/tests ``` -Every test reads the SPDX list from `ISAACCAPTURE_SBOM_LICENSE_DATA`, else -`build/_deps/license-list-data-src/json`; the file API tests also read -`ISAACCAPTURE_SBOM_BUILD_DIR`, else `build/`. A default that holds nothing skips them; a -variable that names a path holding nothing fails them. With `BUILD_TESTING`, [`CMakeLists.txt`](CMakeLists.txt) registers each -test file with CTest as `sbom_` under the label `sbom`, pointing both variables at -the current build. +They read the file API codemodel, and the SPDX list where `build` finds it without +`--license-data`, from the build tree `ISAACCAPTURE_SBOM_BUILD_DIR` names, else `build/`. +An unnamed `build/` that lacks them skips the tests; a named one fails them. CI runs this +command, naming its build, before generating any SBOM. ## How the inventory is worked out diff --git a/scripts/sbom/tests/conftest.py b/scripts/sbom/tests/conftest.py index 8b502e0797..8b00ad66a5 100644 --- a/scripts/sbom/tests/conftest.py +++ b/scripts/sbom/tests/conftest.py @@ -10,9 +10,8 @@ import pytest import synth -from isaaccapture_sbom import discovery, licensing +from isaaccapture_sbom import build, discovery, licensing -LICENSE_DATA_ENV = "ISAACCAPTURE_SBOM_LICENSE_DATA" BUILD_DIR_ENV = "ISAACCAPTURE_SBOM_BUILD_DIR" @@ -24,21 +23,21 @@ def repo_root() -> Path | None: return None -def _located(env: str, usable, missing: str, *in_checkout: str) -> Path: - """The path `env` names, else the checkout's default, if `usable` accepts it. +def _located(usable, missing: str, within=lambda build_dir: build_dir) -> Path: + """`within` the build tree BUILD_DIR_ENV names, else the checkout's `build/`, if `usable`. - A path the environment names is one CTest promised, so its absence fails - every test; without one, the tests skip, since nothing was promised. + A build the environment names is one the caller promised, so its absence + fails every test; without one, the tests skip, since nothing was promised. """ - if os.environ.get(env): - named = Path(os.environ[env]) + if os.environ.get(BUILD_DIR_ENV): + named = within(Path(os.environ[BUILD_DIR_ENV]).resolve()) if usable(named): return named - pytest.fail(f"{env}={named}: {missing}") + pytest.fail(f"{BUILD_DIR_ENV}: {missing} at {named}") root = repo_root() - if root is not None and usable(root.joinpath(*in_checkout)): - return root.joinpath(*in_checkout) - pytest.skip(f"{missing}; configure this project, or set {env}") + if root is not None and usable(found := within(root / "build")): + return found + pytest.skip(f"{missing}; configure this project, or set {BUILD_DIR_ENV}") def _corpus_loads(path: Path) -> bool: @@ -57,13 +56,7 @@ def _has_codemodel(path: Path) -> bool: def license_data(): """The SPDX license list deps/third_party fetched into this build's _deps.""" return _located( - LICENSE_DATA_ENV, - _corpus_loads, - "no SPDX license list data there", - "build", - "_deps", - "license-list-data-src", - "json", + _corpus_loads, "no SPDX license list data", build.default_license_data ) @@ -105,9 +98,7 @@ def live_build_dir(): the anchor to what it actually emits, so a change to the codemodel's shape fails here instead of passing everywhere. """ - return _located( - BUILD_DIR_ENV, _has_codemodel, "no CMake file API codemodel there", "build" - ) + return _located(_has_codemodel, "no CMake file API codemodel") @pytest.fixture From 56730dfef0eb3570ca2365ca57beeb4cd4f33733 Mon Sep 17 00:00:00 2001 From: Andrei Aristarkhov Date: Wed, 21 Oct 2026 11:37:59 -0700 Subject: [PATCH 097/101] ci(sbom): test the collector in a job of its own, beside the build The collector's tests need a configured tree, not a built one, so they no longer wait on the product build. A job per arch configures the product as Release, without plugins, examples, viz or bindings, and runs the tests under every Python a Release leg runs the collector with, reporting each one that fails. wheel-gate needs the job, so nothing publishes unless it passes. SBOM generation in the Release legs no longer waits for it. Signed-off-by: Andrei Aristarkhov --- .github/workflows/build-ubuntu.yml | 70 +++++++++++++++++++++++++----- scripts/sbom/README.md | 5 ++- 2 files changed, 63 insertions(+), 12 deletions(-) diff --git a/.github/workflows/build-ubuntu.yml b/.github/workflows/build-ubuntu.yml index c998d0e028..5e3cb3bdb6 100644 --- a/.github/workflows/build-ubuntu.yml +++ b/.github/workflows/build-ubuntu.yml @@ -176,16 +176,6 @@ jobs: - name: Run All Tests run: ctest --test-dir "${BUILD_DIR}" --output-on-failure --parallel - # The collector's own tests gate SBOM generation, which only Release legs do. - # Naming the build makes a missing input fail the tests, not skip them. - - name: Test the SBOM collector (Release only) - if: matrix.build_type == 'Release' - env: - ISAACCAPTURE_SBOM_BUILD_DIR: ${{ env.BUILD_DIR }} - run: | - uv run --project scripts/sbom --python ${{ matrix.python_version }} \ - --extra dev pytest scripts/sbom/tests - - name: Generate coverage report if: ${{ matrix.build_type == 'Debug' && matrix.python_version == '3.11' && matrix.arch == 'x64' }} run: | @@ -380,6 +370,65 @@ jobs: if-no-files-found: error retention-days: 7 + # The SBOM collector's tests need only a configured tree, so they run beside + # the product build; wheel-gate waits on them. + test-sbom-collector: + runs-on: ${{ matrix.arch == 'arm64' && 'ubuntu-22.04-arm' || 'ubuntu-22.04' }} + permissions: + contents: read + timeout-minutes: 20 + strategy: + fail-fast: ${{ github.event_name == 'pull_request' }} + matrix: + arch: ['x64', 'arm64'] + env: + BUILD_DIR: build + + steps: + - name: Checkout code + uses: actions/checkout@v6 + with: + fetch-depth: 0 # configure derives the version from history + persist-credentials: false + + - name: Install uv + uses: ./.github/actions/setup-uv + + - name: Install Apt dependencies + run: | + sudo apt-get update + sudo apt-get install -y $CORE_APT_DEPS + + # BUILD_TESTING=ON: a test target links a static library the project builds, + # which the codemodel test looks for. + - name: Configure CMake + run: | + cmake -B "${BUILD_DIR}" \ + -DCMAKE_BUILD_TYPE=Release \ + -DBUILD_PLUGINS=OFF \ + -DBUILD_EXAMPLES=OFF \ + -DBUILD_VIZ=OFF \ + -DBUILD_PYTHON_BINDINGS=OFF \ + -DENABLE_CLANG_FORMAT_CHECK=OFF \ + -DBUILD_TESTING=ON + + # Pythons: build-ubuntu's python_version matrix; keep the two lists in step. + # The named build makes missing inputs fail the tests rather than skip them. + - name: Test the SBOM collector + env: + ISAACCAPTURE_SBOM_BUILD_DIR: ${{ env.BUILD_DIR }} + run: | + set -uo pipefail + failed=() + for python in 3.11 3.12 3.13; do + uv run --project scripts/sbom --python "${python}" \ + --extra dev pytest scripts/sbom/tests || failed+=("${python}") + done + if (( ${#failed[@]} )); then + echo "::error::SBOM collector tests failed under Python ${failed[*]}" + exit 1 + fi + pip-editable-install: # Validates the scikit-build-core (PEP 517/660) from-source install path added # for issue #735: an sdist build and `pip install -e .` over the root @@ -1076,6 +1125,7 @@ jobs: permissions: {} needs: - build-ubuntu + - test-sbom-collector - test-viz-gpu - test-viz-sanitizers - test-cloudxr diff --git a/scripts/sbom/README.md b/scripts/sbom/README.md index cdb2782d5c..d64dcdf1b6 100644 --- a/scripts/sbom/README.md +++ b/scripts/sbom/README.md @@ -75,8 +75,9 @@ uv run --project scripts/sbom --extra dev pytest scripts/sbom/tests They read the file API codemodel, and the SPDX list where `build` finds it without `--license-data`, from the build tree `ISAACCAPTURE_SBOM_BUILD_DIR` names, else `build/`. -An unnamed `build/` that lacks them skips the tests; a named one fails them. CI runs this -command, naming its build, before generating any SBOM. +An unnamed `build/` that lacks them skips the tests; a named one fails them. A configured +tree is enough, so CI runs this command in a job of its own, beside the product build, and +publishes nothing unless it passes. ## How the inventory is worked out From 18877f0e8b40275fbcecd51f9253143d5e2698ee Mon Sep 17 00:00:00 2001 From: Andrei Aristarkhov Date: Wed, 21 Oct 2026 14:57:07 -0700 Subject: [PATCH 098/101] fix(sbom): a wheel's licence report lists what the wheel holds, not the build The licence report listed every component the build resolved that reached no file in the wheel. That list describes the build, so the metadata-only isaacteleop wheel -- byte-identical in every CI leg -- got one report on x64 and another on arm64, each naming that arch's CloudXR SDK archives, and a release published both. Drop it from licenses.json and licenses.md, and drop the copy build evidence kept as excluded_components: each entry of its components map already says in_this_wheel and kind, which is the same record. The report also stops copying license_data.source, the build machine's path to the SPDX list; the build evidence keeps it. Signed-off-by: Andrei Aristarkhov --- docs/source/references/sbom.rst | 10 +++++---- scripts/sbom/README.md | 4 ++-- scripts/sbom/isaaccapture_sbom/evidence.py | 1 - scripts/sbom/isaaccapture_sbom/inventory.py | 15 -------------- scripts/sbom/isaaccapture_sbom/report.py | 20 ++++++------------ scripts/sbom/tests/synth.py | 1 - scripts/sbom/tests/test_static_archives.py | 10 +++------ scripts/sbom/tests/test_wheel_sbom.py | 23 ++++++++++++++------- 8 files changed, 33 insertions(+), 51 deletions(-) diff --git a/docs/source/references/sbom.rst b/docs/source/references/sbom.rst index 2273ecf8d0..4acba5d692 100644 --- a/docs/source/references/sbom.rst +++ b/docs/source/references/sbom.rst @@ -29,7 +29,8 @@ What a wheel carries - **The SBOM** lists every file in the wheel but ``RECORD`` and itself: where it came from, under what license, and its digest. It describes the wheel's own contents only. Build - tools (listed in the report under *Present in this build but not in this wheel*), + tools, dependencies this wheel does not carry (the build evidence records them with + ``"in_this_wheel": false``), install-time dependencies (``Requires-Dist`` stays authoritative; the SBOM records the requirement, never a license for it), source distributions and ``pip install .``, containers and the web client are not covered. @@ -49,12 +50,13 @@ Each tagged GitHub release carries, for every wheel: ``.licenses.md`` / ``.licenses.json`` Each redistributed component, the license concluded for it and the file it was read - from, how it reaches the wheel, what the installation environment supplies instead, and - what was excluded and why. + from, how it reaches the wheel, and what the installation environment supplies instead. + It covers this wheel only; what the build used but did not ship is in the build evidence. ``.build-evidence.json`` Resolved commits, declared constraints, archive digests and build options, from the - build machine. + build machine, and every component the build resolved, whether or not it reached the + wheel. and, once per release: diff --git a/scripts/sbom/README.md b/scripts/sbom/README.md index d64dcdf1b6..2ccf403eb3 100644 --- a/scripts/sbom/README.md +++ b/scripts/sbom/README.md @@ -247,8 +247,8 @@ maintained by hand. [`deps/third_party/CMakeLists.txt`](../../deps/third_party/CMakeLists.txt) fetches `spdx/license-list-data` at a pinned commit into `_deps/license-list-data-src`, like any -other dependency, so the collector reports it as a component fetched but shipping -nothing. Each report records the commit its identifications were made against. +other dependency, so the build evidence records it with `"in_this_wheel": false`. Each +report records the commit its identifications were made against. Every current identifier is available to match against. Naming one in the document is a separate question: the document states the license list it was built against, and diff --git a/scripts/sbom/isaaccapture_sbom/evidence.py b/scripts/sbom/isaaccapture_sbom/evidence.py index 66c8e0f6de..89d1510e14 100644 --- a/scripts/sbom/isaaccapture_sbom/evidence.py +++ b/scripts/sbom/isaaccapture_sbom/evidence.py @@ -422,5 +422,4 @@ def document( }, "link_artifacts": sorted(artifacts, key=lambda item: item["output"]), "attributions": [item.as_json() for item in inventory.attributions.values()], - "excluded_components": inventory.absent_components, } diff --git a/scripts/sbom/isaaccapture_sbom/inventory.py b/scripts/sbom/isaaccapture_sbom/inventory.py index 8e15ac2587..928dba0f47 100644 --- a/scripts/sbom/isaaccapture_sbom/inventory.py +++ b/scripts/sbom/isaaccapture_sbom/inventory.py @@ -37,13 +37,6 @@ if TYPE_CHECKING: # pragma: no cover from .evidence import Discovery -# How a component that reached no file got into the build, by kind. Checked-in -# sources were never fetched, and the collector's own vocabulary says so. -_ARRIVED = { - "vendored-source": "checked into this repository", - "vcpkg-port": "installed by vcpkg for this build", -} - # Text-ish members worth reading a REUSE tag out of. _TAGGABLE = { ".py", @@ -107,7 +100,6 @@ class Inventory: unattributed: list[str] = field(default_factory=list) # Link-line inputs nothing explains, as ` (linked by ...): `. untraced: list[str] = field(default_factory=list) - absent_components: dict[str, str] = field(default_factory=dict) _indexed: tuple[dict, dict] | None = field(default=None, repr=False) def files_of(self, key: str) -> list[str]: @@ -780,12 +772,6 @@ def build( for key in sorted(roles) if key in resolver.components } - absent = { - key: _ARRIVED.get(resolver.components[key].kind, "fetched for this build") - + "; contributes to no file in this wheel" - for key in sorted(resolver.components) - if key not in roles - } external: dict[str, list[str]] = {} for entry in wheel.entries: @@ -808,7 +794,6 @@ def build( for path, record in resolver.static_inputs.items() if not record["explained"] ), - absent_components=absent, ) diff --git a/scripts/sbom/isaaccapture_sbom/report.py b/scripts/sbom/isaaccapture_sbom/report.py index 88f3c1fdc6..5618eb621a 100644 --- a/scripts/sbom/isaaccapture_sbom/report.py +++ b/scripts/sbom/isaaccapture_sbom/report.py @@ -291,17 +291,18 @@ def report(inventory: Inventory, evidence_doc: dict, wheel_name: str) -> dict: evidence_doc.get("project", {}).get("describe") or "" ).endswith("-dirty"), "spdx_license_list_version": evidence_doc.get("license_list_version"), - "license_data": evidence_doc.get("license_data", {}), + # Where the build read the list from is the build evidence's to record. + "license_data": { + key: value + for key, value in evidence_doc.get("license_data", {}).items() + if key != "source" + }, "components": components, "externally_supplied_runtime": [ {"soname": soname, "required_by": consumers} for soname, consumers in inventory.external_runtime.items() ], "consumer_requirements": inventory.requires_dist, - "exclusions": [ - {"component": key, "reason": reason} - for key, reason in sorted(inventory.absent_components.items()) - ], **gaps, } @@ -391,15 +392,6 @@ def report_markdown(payload: dict) -> str: "None declared." ] - lines += ["", "## Present in this build but not in this wheel", ""] - if payload["exclusions"]: - lines += [ - f"- `{item['component']}` — {item['reason']}" - for item in payload["exclusions"] - ] - else: - lines.append("None.") - lines += ["", "## License evidence gaps", ""] listed = [key for key in GAP_LISTS if payload[key]] for key in listed: diff --git a/scripts/sbom/tests/synth.py b/scripts/sbom/tests/synth.py index 8e8c605843..80c3655554 100644 --- a/scripts/sbom/tests/synth.py +++ b/scripts/sbom/tests/synth.py @@ -1084,7 +1084,6 @@ def collect_and_bind(workspace: Workspace, **options) -> dict: "manifest_path": manifest_path, "evidence_path": out_dir / f"{STEM}.build-evidence.json", "spdx_path": out_dir / f"{STEM}.spdx.json", - "report_path": out_dir / f"{STEM}.licenses.json", } diff --git a/scripts/sbom/tests/test_static_archives.py b/scripts/sbom/tests/test_static_archives.py index 4ec5d487be..849e59d212 100644 --- a/scripts/sbom/tests/test_static_archives.py +++ b/scripts/sbom/tests/test_static_archives.py @@ -1179,7 +1179,7 @@ def test_a_file_no_port_lists_is_still_unexplained(workspace, monkeypatch): synth.collect(workspace) -def test_an_unlinked_port_is_excluded_as_installed_by_vcpkg(workspace): +def test_an_unlinked_port_is_recorded_as_a_vcpkg_port_not_in_the_wheel(workspace): synth.vcpkg_port( workspace.build / "vcpkg_installed", "zlib", @@ -1191,12 +1191,8 @@ def test_an_unlinked_port_is_excluded_as_installed_by_vcpkg(workspace): collected = synth.collect(workspace) - exclusion = next( - item - for item in collected["licenses"]["exclusions"] - if item["component"] == "vcpkg:zlib" - ) - assert exclusion["reason"].startswith("installed by vcpkg for this build") + zlib = collected["build-evidence"]["components"]["vcpkg:zlib"] + assert (zlib["kind"], zlib["in_this_wheel"]) == ("vcpkg-port", False) def _eigen_port(installed) -> None: diff --git a/scripts/sbom/tests/test_wheel_sbom.py b/scripts/sbom/tests/test_wheel_sbom.py index c85ecb86bc..2ff9e6cdcb 100644 --- a/scripts/sbom/tests/test_wheel_sbom.py +++ b/scripts/sbom/tests/test_wheel_sbom.py @@ -116,14 +116,24 @@ def test_components_are_discovered_not_declared(built): assert "omega" not in names, "a test-only dependency is not in the wheel" -def test_a_dependency_reaching_no_wheel_file_is_recorded_as_excluded(built): - report = json.loads(built["report_path"].read_text(encoding="utf-8")) - excluded = [item["component"] for item in report["exclusions"]] +def test_a_dependency_reaching_no_wheel_file_is_recorded_by_the_build(built): + components = built["build-evidence"]["components"] - assert "omega" in excluded, "a test-only dependency ships nothing" + assert components["omega"]["in_this_wheel"] is False, "a test-only dependency" # The SPDX reference data is fetched like any other dependency, so it is - # inventoried like any other dependency -- and excluded for the same reason. - assert "license-list-data" in excluded + # inventoried like any other dependency -- and ships nothing for the same reason. + assert components["license-list-data"]["in_this_wheel"] is False + # What ships nothing is the build's to record, not the wheel's. + assert "omega" not in json.dumps(built["licenses"]) + assert "omega" not in built["md"] + + +def test_where_the_license_list_was_read_is_left_to_the_build_evidence(built): + evidence = built["build-evidence"]["license_data"] + report = built["licenses"]["license_data"] + + assert "source" in evidence and "source" not in report + assert report["commit"] == evidence["commit"] def test_licenses_are_packaged_and_declared(built): @@ -254,7 +264,6 @@ def _report_of(**gaps) -> dict: "components": [row], "externally_supplied_runtime": [], "consumer_requirements": [], - "exclusions": [], **{key: gaps.get(key, []) for key in report_module.GAP_LISTS}, } From 6ae270b69472990dae458ba8117dc62d5cd2276d Mon Sep 17 00:00:00 2001 From: Andrei Aristarkhov Date: Wed, 21 Oct 2026 15:21:10 -0700 Subject: [PATCH 099/101] ci(sbom): publish only the evidence a release's manifest binds Every Release matrix entry builds the py3-none-any isaacteleop wheel, so its sidecars reach the release job once per entry. The job's shell loop kept the first copy under its own name and any differing one as .: files no manifest bound and check-set never read, with the plain name chosen by glob order and the bound one by merge_records' own. merge-manifests now refuses one wheel name whose SBOM or license report differs between entries, as it already refused differing wheel bytes; build evidence may differ, and the first entry's is bound. --evidence-out copies exactly the bound sidecars, from beside the record bound, after checking each against its digest, and the workflow's loop goes. check-set fails a file in --evidence-dir the manifest does not name, as it already did a wheel, so what the release uploads and what it verified are one set. Signed-off-by: Andrei Aristarkhov --- .github/workflows/build-ubuntu.yml | 26 +--- docs/source/references/sbom.rst | 4 +- scripts/sbom/README.md | 9 +- scripts/sbom/isaaccapture_sbom/build.py | 80 ++++++++---- scripts/sbom/isaaccapture_sbom/cli.py | 12 +- scripts/sbom/isaaccapture_sbom/validate.py | 48 +++++-- scripts/sbom/tests/test_gates.py | 142 +++++++++++++++++++-- 7 files changed, 244 insertions(+), 77 deletions(-) diff --git a/.github/workflows/build-ubuntu.yml b/.github/workflows/build-ubuntu.yml index 5e3cb3bdb6..7255f8901f 100644 --- a/.github/workflows/build-ubuntu.yml +++ b/.github/workflows/build-ubuntu.yml @@ -1200,32 +1200,14 @@ jobs: - name: Assemble and verify the release evidence set run: | set -euo pipefail - shopt -s nullglob - - mkdir -p release-evidence - # A `py3-none-any` wheel is built by every matrix entry, so its sidecars - # arrive once per entry under the same name. Identical copies are one - # file; differing ones are kept apart under the entry that produced - # them, because dropping one would publish evidence for a wheel nobody - # can tell apart from the five it is not. - for sidecar in sbom-sidecars/*/*.spdx.json \ - sbom-sidecars/*/*.licenses.json \ - sbom-sidecars/*/*.licenses.md \ - sbom-sidecars/*/*.build-evidence.json; do - name="$(basename "${sidecar}")" - entry="$(basename "$(dirname "${sidecar}")")" - if [[ ! -e "release-evidence/${name}" ]]; then - cp "${sidecar}" "release-evidence/${name}" - elif cmp -s "${sidecar}" "release-evidence/${name}"; then - continue - else - cp "${sidecar}" "release-evidence/${entry}.${name}" - fi - done + # A `py3-none-any` wheel arrives from every matrix entry; the merge + # refuses one whose copies disagree, and copies only the evidence the + # merged manifest binds. uv run --project scripts/sbom --python 3.12 \ isaaccapture-sbom merge-manifests \ --output release-evidence/isaaccapture-sbom-manifest.json \ + --evidence-out release-evidence \ sbom-sidecars/*/manifest.json # Rehearses the published verification instructions against the exact diff --git a/docs/source/references/sbom.rst b/docs/source/references/sbom.rst index 4acba5d692..5355f797dd 100644 --- a/docs/source/references/sbom.rst +++ b/docs/source/references/sbom.rst @@ -75,8 +75,8 @@ You need ``uv`` and a clone of this repository: uv run --project IsaacCapture/scripts/sbom isaaccapture-sbom check \ --wheel isaaccapture---.whl -To check a whole release, put its wheels and evidence files in a directory of their own, -and keep other wheels out of it: +To check a whole release, put its wheels and evidence files in a directory of their own; +any other file there fails the check: .. code-block:: bash diff --git a/scripts/sbom/README.md b/scripts/sbom/README.md index 2ccf403eb3..fa98988f76 100644 --- a/scripts/sbom/README.md +++ b/scripts/sbom/README.md @@ -61,9 +61,12 @@ lists every member except `RECORD` and itself, since neither can carry its own d `--build-evidence` also checks that the build accounts for every member; `--manifest` checks the wheel against the digest bound at publication and the sidecars beside it. `check-set` runs the whole thing over every wheel in a release, and fails a wheel in -`--wheel-dir` that the manifest does not list. Any command reading a manifest refuses -one that lists no wheel. CI validates each published document with `spdx-tools` before -release. +`--wheel-dir`, or a file in `--evidence-dir`, that the manifest does not name. +`merge-manifests` refuses a wheel name whose builds produced different bytes, SBOMs or +license reports. Only build evidence may differ; the first manifest's in path order (in +CI, artifact-name order) is bound, and `--evidence-out` copies exactly the bound files. +Any command reading a manifest refuses one that lists no wheel. CI validates each +published document with `spdx-tools` before release. ### Tests diff --git a/scripts/sbom/isaaccapture_sbom/build.py b/scripts/sbom/isaaccapture_sbom/build.py index ae90c80e6f..9e24d99980 100644 --- a/scripts/sbom/isaaccapture_sbom/build.py +++ b/scripts/sbom/isaaccapture_sbom/build.py @@ -511,44 +511,78 @@ def build( } -def merge_manifests(paths: list[Path]) -> dict: - """Fold per-variant manifests into the one a release advertises.""" - return merge_records( - [ - item - for path in sorted(paths) - for item in validate_module.manifest_records(path) - ] - ) +def merge_manifests(paths: list[Path], evidence_out: Path | None = None) -> dict: + """Fold per-variant manifests into the one a release advertises. + + With `evidence_out`, copy there the sidecars it binds, from beside the + manifest that named them, and nothing else. + """ + found = [ + (path.parent, item) + for path in sorted(paths) + for item in validate_module.manifest_records(path) + ] + merged = merge_records([item for _, item in found]) + if evidence_out is None: + return merged + + # From beside the very record the merge bound, so the copy cannot follow a + # different "first" than the binding did. + beside = {id(item): parent for parent, item in found} + bound = [(beside[id(record)], record) for record in merged["wheels"]] + # Checked before anything is written, so a refused set copies nothing. + failures = [ + f"{record['filename']}: {failure}" + for parent, record in bound + for failure in validate_module.sidecar_failures(parent, record) + ] + if failures: + raise BuildError("; ".join(failures)) + evidence_out.mkdir(parents=True, exist_ok=True) + for parent, record in bound: + # Already in place, and verified above; copying onto itself would raise. + if parent.resolve() == evidence_out.resolve(): + continue + for sidecar in record["sidecars"].values(): + shutil.copyfile( + parent / sidecar["filename"], evidence_out / sidecar["filename"] + ) + return merged def merge_records(wheels: list[dict]) -> dict: """The manifest for a set of wheels, stamped when the set was complete.""" # A `py3-none-any` wheel is built by every matrix entry, so a release set - # names it once per entry. What must agree is the wheel: one filename - # standing for two different artifacts is the thing a consumer cannot tell - # apart. The sidecars may legitimately differ -- build evidence describes - # the build, and six builds produced this wheel -- so the first entry's - # evidence is the one the manifest binds, matching the set assembled beside - # it. + # names it once per entry. The wheel, and whatever is derived from it alone, + # must agree, or one filename stands for things a consumer cannot tell apart. + # Build evidence describes the build, and every entry built this wheel, so + # the first entry's is the one the manifest binds. identity = ("filename", "sha256", "size", "sbom_in_wheel") unique: dict[str, dict] = {} conflicting: set[str] = set() + disagreeing: set[str] = set() for item in wheels: - name = item["filename"] - seen = unique.get(name) - if seen is not None: - if [seen.get(key) for key in identity] != [ - item.get(key) for key in identity - ]: - conflicting.add(name) + validate_module.checked_record(item) + seen = unique.setdefault(item["filename"], item) + if seen is item: continue - unique[name] = item + if [seen.get(key) for key in identity] != [item.get(key) for key in identity]: + conflicting.add(item["filename"]) + disagreeing.update( + f"{item['filename']} ({kind})" + for kind in validate_module.CONTENT_SIDECARS + if seen["sidecars"][kind]["sha256"] != item["sidecars"][kind]["sha256"] + ) if conflicting: raise BuildError( "the same wheel filename was published with different contents: " + ", ".join(sorted(conflicting)) ) + if disagreeing: + raise BuildError( + "builds of one wheel described it differently: " + + ", ".join(sorted(disagreeing)) + ) return { "schema": "isaaccapture-sbom-manifest/1", "generated_at": stamped_now().strftime("%Y-%m-%dT%H:%M:%SZ"), diff --git a/scripts/sbom/isaaccapture_sbom/cli.py b/scripts/sbom/isaaccapture_sbom/cli.py index 4b18f285bd..927deafe66 100644 --- a/scripts/sbom/isaaccapture_sbom/cli.py +++ b/scripts/sbom/isaaccapture_sbom/cli.py @@ -115,7 +115,10 @@ def _cmd_check_set(args: argparse.Namespace) -> int: def _cmd_merge_manifests(args: argparse.Namespace) -> int: - merged = build_module.merge_manifests([Path(item) for item in args.manifest]) + merged = build_module.merge_manifests( + [Path(item) for item in args.manifest], + Path(args.evidence_out) if args.evidence_out else None, + ) Path(args.output).write_text( json.dumps(merged, indent=2, sort_keys=True) + "\n", encoding="utf-8" ) @@ -187,7 +190,9 @@ def main(argv: list[str] | None = None) -> int: help="the release's wheels; one the manifest does not list fails the set", ) check_set.add_argument( - "--evidence-dir", required=True, help="where the release's sidecars live" + "--evidence-dir", + required=True, + help="the release's sidecars; a file the manifest does not bind fails the set", ) check_set.set_defaults(handler=_cmd_check_set) @@ -195,6 +200,9 @@ def main(argv: list[str] | None = None) -> int: "merge-manifests", help="fold per-variant manifests into one" ) merge.add_argument("--output", required=True, help="merged manifest to write") + merge.add_argument( + "--evidence-out", help="copy the sidecars the merged manifest binds here" + ) merge.add_argument("manifest", nargs="+", help="per-variant manifests") merge.set_defaults(handler=_cmd_merge_manifests) diff --git a/scripts/sbom/isaaccapture_sbom/validate.py b/scripts/sbom/isaaccapture_sbom/validate.py index ccc7fc7c1a..9b2b4a8145 100644 --- a/scripts/sbom/isaaccapture_sbom/validate.py +++ b/scripts/sbom/isaaccapture_sbom/validate.py @@ -22,7 +22,9 @@ from . import wheelfile -_SIDECAR_KINDS = ("spdx", "build_evidence", "license_report_json", "license_report_md") +# Derived from the wheel alone, so every build of one wheel writes the same bytes. +CONTENT_SIDECARS = ("spdx", "license_report_json", "license_report_md") +_SIDECAR_KINDS = (*CONTENT_SIDECARS, "build_evidence") class ManifestError(Exception): @@ -63,6 +65,18 @@ def manifest_records(path: Path) -> list[dict]: return records +def sidecar_failures(beside: Path, record: dict) -> list[str]: + """Each sidecar a checked record binds that is not beside it as bound.""" + failures = [] + for kind, sidecar in record["sidecars"].items(): + path = beside / sidecar["filename"] + if not path.is_file(): + failures.append(f"sidecar {kind} is missing: {sidecar['filename']}") + elif sha256_file(path) != sidecar["sha256"]: + failures.append(f"sidecar {kind} does not match its recorded digest") + return failures + + def _is_bare_name(name: str) -> bool: """A file name joined onto a directory must stay inside that directory.""" return name not in ("", ".", "..") and not any(sep in name for sep in "/\\:") @@ -73,10 +87,10 @@ def _record_for(records: list[dict], filename: str) -> dict: seen = [item for item in records if item.get("filename") == filename] if len(seen) > 1: raise ManifestError(f"{filename} is advertised {len(seen)} times") - return _checked_record(seen[0]) if seen else {} + return checked_record(seen[0]) if seen else {} -def _checked_record(record: dict) -> dict: +def checked_record(record: dict) -> dict: """A record holding every field the checks read, or ManifestError.""" filename = record["filename"] for key in ("sha256", "size", "sbom_in_wheel", "sidecars"): @@ -591,12 +605,7 @@ def _check_manifest( ) beside = evidence_dir if evidence_dir is not None else manifest_path.parent - for kind, sidecar in record["sidecars"].items(): - path = beside / sidecar["filename"] - if not path.is_file(): - failures.append(f"sidecar {kind} is missing: {sidecar['filename']}") - elif sha256_file(path) != sidecar["sha256"]: - failures.append(f"sidecar {kind} does not match its recorded digest") + failures.extend(sidecar_failures(beside, record)) published = beside / record["sidecars"]["spdx"]["filename"] if published.is_file(): @@ -670,13 +679,14 @@ def _cross_document_refs( def check_set(manifest_path: Path, wheel_dir: Path, evidence_dir: Path) -> list[str]: - """Run :func:`check` over every wheel a merged manifest advertises.""" + """Run :func:`check` over every wheel a merged manifest advertises, and fail + any wheel or evidence file here that it does not name.""" failures: list[str] = [] records = manifest_records(manifest_path) for name, seen in sorted(Counter(item["filename"] for item in records).items()): if seen > 1: raise ManifestError(f"{name} is advertised {seen} times") - by_name = {item["filename"]: _checked_record(item) for item in records} + by_name = {item["filename"]: checked_record(item) for item in records} # Every wheel here is published with the set, so one the manifest omits # would go out unchecked. @@ -684,6 +694,22 @@ def check_set(manifest_path: Path, wheel_dir: Path, evidence_dir: Path) -> list[ failures.append( f"{stray} is in {wheel_dir}, and {manifest_path.name} does not list it" ) + # The same holds for evidence: a file no record binds is never verified. + # Wheels sharing the directory are the check above's to judge. + expected = { + sidecar["filename"] + for record in by_name.values() + for sidecar in record["sidecars"].values() + } + if manifest_path.parent.resolve() == evidence_dir.resolve(): + expected.add(manifest_path.name) + if evidence_dir.resolve() == wheel_dir.resolve(): + expected |= {item.name for item in wheel_dir.glob("*.whl")} + present = {item.name for item in evidence_dir.glob("*") if item.is_file()} + for stray in sorted(present - expected): + failures.append( + f"{stray} is in {evidence_dir}, and {manifest_path.name} does not bind it" + ) # The whole release is here, which is the only place a reference from one # document to another can be resolved rather than merely declared. diff --git a/scripts/sbom/tests/test_gates.py b/scripts/sbom/tests/test_gates.py index 41957ea362..00e36b6e87 100644 --- a/scripts/sbom/tests/test_gates.py +++ b/scripts/sbom/tests/test_gates.py @@ -723,30 +723,144 @@ def test_a_third_partys_licence_does_not_discharge_this_one(workspace, license_d ) == ["Apache-2.0"] -def test_one_wheel_with_differing_evidence_merges(tmp_path, built): - """Six builds of a universal wheel produce six build-evidence sidecars. - - The wheel is the same artifact; the evidence describes the build that made - it, and those differ. A release names one evidence set, and refusing the - whole release over that would block every build of a `py3-none-any` wheel. - """ +@pytest.mark.parametrize("kind", [*validate_module.CONTENT_SIDECARS, "build_evidence"]) +def test_builds_of_one_wheel_may_differ_only_in_build_evidence(tmp_path, built, kind): + """Every entry builds a `py3-none-any` wheel. Its evidence describes each + build, so it may differ and the first entry's is bound; what is derived + from the wheel alone may not.""" first = tmp_path / "a.json" second = tmp_path / "b.json" first.write_text(json.dumps(built["manifest"]), encoding="utf-8") other = json.loads(json.dumps(built["manifest"])) - other["wheels"][0]["sidecars"]["build_evidence"]["sha256"] = "1" * 64 + other["wheels"][0]["sidecars"][kind]["sha256"] = "1" * 64 second.write_text(json.dumps(other), encoding="utf-8") - merged = build_module.merge_manifests([first, second]) + if kind == "build_evidence": + merged = build_module.merge_manifests([first, second]) + assert merged["wheels"] == built["manifest"]["wheels"] + else: + with pytest.raises( + build_module.BuildError, match=rf"described it .*\({kind}\)" + ): + build_module.merge_manifests([first, second]) + + +def _two_builds_of_one_wheel(built, tmp_path): + """Two entries' sidecar directories whose build evidence differs.""" + entries = [] + for name, note in (("a", ""), ("b", "\n")): + entry = tmp_path / "sidecars" / name + shutil.copytree(built["out_dir"], entry) + evidence = entry / built["evidence_path"].name + evidence.write_bytes(evidence.read_bytes() + note.encode()) + manifest = json.loads((entry / "manifest.json").read_text(encoding="utf-8")) + manifest["wheels"][0]["sidecars"]["build_evidence"]["sha256"] = ( + build_module.sha256_file(evidence) + ) + (entry / "manifest.json").write_text(json.dumps(manifest), encoding="utf-8") + entries.append(entry) + return entries + + +def test_a_merged_release_holds_the_evidence_it_binds_and_verifies(built, tmp_path): + first, second = _two_builds_of_one_wheel(built, tmp_path) + release = tmp_path / "release" + wheels = tmp_path / "wheels" + wheels.mkdir() + shutil.copy(built["wheel"], wheels) + + # Named out of order: the first in path order is bound, and copied. + assert _merge_cli([second, first], release) == 0 + + manifest = release / "isaaccapture-sbom-manifest.json" + bound = json.loads(manifest.read_text(encoding="utf-8"))["wheels"][0]["sidecars"] + assert {item.name for item in release.iterdir()} == { + manifest.name, + *(item["filename"] for item in bound.values()), + } + evidence = release / bound["build_evidence"]["filename"] + assert evidence.read_bytes() == (first / evidence.name).read_bytes() + assert validate_module.check_set(manifest, wheels, release) == [] + + +def test_evidence_the_manifest_does_not_bind_fails_the_release_set(built, tmp_path): + merged, wheels, evidence = _release_layout(built, tmp_path) + stray = evidence / f"b.{built['evidence_path'].name}" + stray.write_bytes(built["evidence_path"].read_bytes()) + + failures = validate_module.check_set(merged, wheels, evidence) + + assert failures == [ + f"{stray.name} is in {evidence}, and {merged.name} does not bind it" + ] + + +def test_a_file_named_like_a_manifest_kept_elsewhere_is_stray(built, tmp_path): + merged, wheels, evidence = _release_layout(built, tmp_path) + stray = evidence / merged.name + stray.write_text("{}", encoding="utf-8") + + assert validate_module.check_set(merged, wheels, evidence) == [ + f"{stray.name} is in {evidence}, and {merged.name} does not bind it" + ] + + +def test_merging_into_the_directory_holding_the_evidence(built, tmp_path): + entry = tmp_path / "entry" + shutil.copytree(built["out_dir"], entry) + + assert _merge_cli([entry], entry) == 0 - assert len(merged["wheels"]) == 1 - # The first entry's evidence, matching the set assembled alongside it. - assert ( - merged["wheels"][0]["sidecars"]["build_evidence"]["sha256"] - == (built["manifest"]["wheels"][0]["sidecars"]["build_evidence"]["sha256"]) + +def test_a_release_checked_from_one_directory_verifies(built, tmp_path): + """The documented recipe: wheels, evidence and manifest side by side.""" + merged, wheels, evidence = _release_layout(built, tmp_path) + shutil.copy(next(wheels.iterdir()), evidence) + shutil.copy(merged, evidence) + + manifest = evidence / merged.name + assert validate_module.check_set(manifest, evidence, evidence) == [] + + stray = evidence / "isaacteleop-0.4.0-py3-none-any.whl" + shutil.copy(built["wheel"], stray) + assert validate_module.check_set(manifest, evidence, evidence) == [ + f"{stray.name} is in {evidence}, and {manifest.name} does not list it" + ] + + +def _merge_cli(entries, release): + return cli.main( + [ + "merge-manifests", + "--output", + str(release / "isaaccapture-sbom-manifest.json"), + "--evidence-out", + str(release), + *(str(entry / "manifest.json") for entry in entries), + ] ) +def test_a_missing_bound_sidecar_copies_nothing(built, tmp_path, capsys): + first, second = _two_builds_of_one_wheel(built, tmp_path) + (first / built["spdx_path"].name).unlink() + release = tmp_path / "release" + + assert _merge_cli([first, second], release) == 2 + assert "sidecar spdx is missing" in capsys.readouterr().err + assert not release.exists() + + +def test_merge_manifests_refuses_malformed_sidecars(tmp_path): + record = _record() + record["sidecars"] = list(record["sidecars"]) + + with pytest.raises( + validate_module.ManifestError, match="sidecars must each name a file" + ): + build_module.merge_manifests([_manifest_of(tmp_path, [record])]) + + def test_reuse_toml_declares_the_terms_of_an_untagged_vendored_file(workspace): """A checked-in header that names no identifier, as Chromium's key table does, under `src/vendor` so the extension that includes it ships it.""" From a1d6d68caaef4720446365c2e9aea029fb0a0f64 Mon Sep 17 00:00:00 2001 From: Andrei Aristarkhov Date: Thu, 22 Oct 2026 11:07:25 -0700 Subject: [PATCH 100/101] ci(sbom): attach a release's evidence as one reproducible archive A tag rehearsal showed GitHub renames release assets: the epoch wheel isaacteleop-1!X.Y.Z's sidecars were published as isaacteleop-1.X.Y.Z-*, so check-set on the downloaded release found four files missing and four it did not bind. Names inside a zip are kept as written. The release job pins the clock to the commit, merges straight into an isaaccapture-sbom-/ directory, and zips it with fixed times, modes and order and no extra fields, so a rerun writes the same bytes. It runs check-set on what unpacking the zip gives, as the docs tell consumers to, and attaches only the zip. The manifest's bound file names get one definition, shared by merge-manifests and check-set. Signed-off-by: Andrei Aristarkhov --- .github/workflows/build-ubuntu.yml | 41 +++++++++++++++------- docs/source/references/sbom.rst | 13 ++++--- scripts/sbom/isaaccapture_sbom/build.py | 6 ++-- scripts/sbom/isaaccapture_sbom/validate.py | 15 +++++--- 4 files changed, 49 insertions(+), 26 deletions(-) diff --git a/.github/workflows/build-ubuntu.yml b/.github/workflows/build-ubuntu.yml index 7255f8901f..b996f23b5d 100644 --- a/.github/workflows/build-ubuntu.yml +++ b/.github/workflows/build-ubuntu.yml @@ -1164,6 +1164,8 @@ jobs: permissions: contents: write timeout-minutes: 20 + env: + ARCHIVE: isaaccapture-sbom-${{ github.ref_name }} # `!cancelled()` for the reason spelled out on `publish-wheel`: a skip # anywhere upstream of `wheel-gate` reaches here too, and the result check @@ -1197,7 +1199,12 @@ jobs: pattern: isaaccapture-sbom-* path: sbom-sidecars - - name: Assemble and verify the release evidence set + # The merged manifest and the archive are stamped with this, so re-running + # this job publishes the same bytes. + - name: Pin the build clock to the commit + run: echo "SOURCE_DATE_EPOCH=$(git log -1 --pretty=%ct)" >> "$GITHUB_ENV" + + - name: Pack and verify the release evidence archive run: | set -euo pipefail @@ -1206,18 +1213,28 @@ jobs: # merged manifest binds. uv run --project scripts/sbom --python 3.12 \ isaaccapture-sbom merge-manifests \ - --output release-evidence/isaaccapture-sbom-manifest.json \ - --evidence-out release-evidence \ + --output "${ARCHIVE}/isaaccapture-sbom-manifest.json" \ + --evidence-out "${ARCHIVE}" \ sbom-sidecars/*/manifest.json - # Rehearses the published verification instructions against the exact - # bytes a consumer will download, and re-checks every advertised digest. + # One zip, not loose assets: GitHub renames release assets (`!` becomes + # `.`), and names inside a zip are kept. Fixed times, modes and order, + # and no owner or extra time fields (-X), so a rerun zips the same + # bytes; zip stores local time, hence TZ. + touch -d "@${SOURCE_DATE_EPOCH}" "${ARCHIVE}"/* + chmod 644 "${ARCHIVE}"/* + printf '%s\n' "${ARCHIVE}"/* | LC_ALL=C sort \ + | TZ=UTC zip -X -q "${ARCHIVE}.zip" -@ + + # Rehearses the published verification instructions on what a consumer + # unpacks, and re-checks every advertised digest. + unzip -q "${ARCHIVE}.zip" -d unpacked uv run --project scripts/sbom --python 3.12 \ isaaccapture-sbom check-set \ - --manifest release-evidence/isaaccapture-sbom-manifest.json \ - --wheel-dir wheels --evidence-dir release-evidence + --manifest "unpacked/${ARCHIVE}/isaaccapture-sbom-manifest.json" \ + --wheel-dir wheels --evidence-dir "unpacked/${ARCHIVE}" - ls -la release-evidence + unzip -l "${ARCHIVE}.zip" - name: Attach evidence to the release env: @@ -1239,11 +1256,11 @@ jobs: sleep 20 done - # --clobber makes a retry after a partial upload idempotent. Draft state - # and release notes are left exactly as they were. - gh release upload "${TAG}" release-evidence/* --clobber + # --clobber makes a retry idempotent. Draft state and release notes are + # left exactly as they were. + gh release upload "${TAG}" "${ARCHIVE}.zip" --clobber - echo "Attached $(ls release-evidence | wc -l) evidence file(s) to ${TAG}" + echo "Attached ${ARCHIVE}.zip to ${TAG}" publish-wheel: # to internal Artifactory runs-on: [self-hosted, linux] diff --git a/docs/source/references/sbom.rst b/docs/source/references/sbom.rst index 5355f797dd..6b1af9cbf4 100644 --- a/docs/source/references/sbom.rst +++ b/docs/source/references/sbom.rst @@ -43,7 +43,8 @@ What a wheel carries Evidence published with a release --------------------------------- -Each tagged GitHub release carries, for every wheel: +Each tagged GitHub release carries one archive, ``isaaccapture-sbom-.zip``. Unpacked, +its ``isaaccapture-sbom-/`` directory holds, for every wheel: ``.spdx.json`` An exact copy of the document embedded in the wheel. @@ -75,14 +76,16 @@ You need ``uv`` and a clone of this repository: uv run --project IsaacCapture/scripts/sbom isaaccapture-sbom check \ --wheel isaaccapture---.whl -To check a whole release, put its wheels and evidence files in a directory of their own; -any other file there fails the check: +To check a whole release, unpack its archive and put every wheel its manifest lists, for +each Python version and architecture, in a directory of their own. A wheel or evidence file +the manifest does not name fails the check: .. code-block:: bash + unzip isaaccapture-sbom-.zip uv run --project IsaacCapture/scripts/sbom isaaccapture-sbom check-set \ - --manifest release/isaaccapture-sbom-manifest.json \ - --wheel-dir release --evidence-dir release + --manifest isaaccapture-sbom-/isaaccapture-sbom-manifest.json \ + --wheel-dir wheels --evidence-dir isaaccapture-sbom- ``check`` confirms the SBOM accounts for every file, its digests match the wheel and its ``RECORD``, each license text it cites is packaged where it says, and every diff --git a/scripts/sbom/isaaccapture_sbom/build.py b/scripts/sbom/isaaccapture_sbom/build.py index 9e24d99980..d034671432 100644 --- a/scripts/sbom/isaaccapture_sbom/build.py +++ b/scripts/sbom/isaaccapture_sbom/build.py @@ -543,10 +543,8 @@ def merge_manifests(paths: list[Path], evidence_out: Path | None = None) -> dict # Already in place, and verified above; copying onto itself would raise. if parent.resolve() == evidence_out.resolve(): continue - for sidecar in record["sidecars"].values(): - shutil.copyfile( - parent / sidecar["filename"], evidence_out / sidecar["filename"] - ) + for name in validate_module.bound_names([record]): + shutil.copyfile(parent / name, evidence_out / name) return merged diff --git a/scripts/sbom/isaaccapture_sbom/validate.py b/scripts/sbom/isaaccapture_sbom/validate.py index 9b2b4a8145..c15fa8612b 100644 --- a/scripts/sbom/isaaccapture_sbom/validate.py +++ b/scripts/sbom/isaaccapture_sbom/validate.py @@ -65,6 +65,15 @@ def manifest_records(path: Path) -> list[dict]: return records +def bound_names(records) -> list[str]: + """The sidecar file names the records bind.""" + return [ + sidecar["filename"] + for record in records + for sidecar in record["sidecars"].values() + ] + + def sidecar_failures(beside: Path, record: dict) -> list[str]: """Each sidecar a checked record binds that is not beside it as bound.""" failures = [] @@ -696,11 +705,7 @@ def check_set(manifest_path: Path, wheel_dir: Path, evidence_dir: Path) -> list[ ) # The same holds for evidence: a file no record binds is never verified. # Wheels sharing the directory are the check above's to judge. - expected = { - sidecar["filename"] - for record in by_name.values() - for sidecar in record["sidecars"].values() - } + expected = set(bound_names(by_name.values())) if manifest_path.parent.resolve() == evidence_dir.resolve(): expected.add(manifest_path.name) if evidence_dir.resolve() == wheel_dir.resolve(): From d5189f4372d561957d6e8179c49dc3209cb4fc4a Mon Sep 17 00:00:00 2001 From: Andrei Aristarkhov Date: Thu, 22 Oct 2026 12:04:01 -0700 Subject: [PATCH 101/101] Trim redundant SBOM comments and docstrings Signed-off-by: Andrei Aristarkhov --- .github/workflows/build-ubuntu.yml | 53 +-- .gitignore | 2 +- CMakeLists.txt | 8 +- deps/cloudxr/.gitignore | 2 +- deps/cloudxr/openxr_extensions/CMakeLists.txt | 4 +- deps/third_party/CMakeLists.txt | 6 +- .../third_party/fetch_license_list_data.cmake | 8 +- .../openxr_extensions/CMakeLists.txt | 9 +- pyproject.toml | 11 +- scripts/download_cloudxr_runtime_sdk.sh | 5 +- scripts/sbom/isaaccapture_sbom/__init__.py | 6 +- scripts/sbom/isaaccapture_sbom/build.py | 127 ++---- scripts/sbom/isaaccapture_sbom/cli.py | 16 +- scripts/sbom/isaaccapture_sbom/discovery.py | 426 +++++------------- scripts/sbom/isaaccapture_sbom/document.py | 111 ++--- scripts/sbom/isaaccapture_sbom/elf.py | 32 +- scripts/sbom/isaaccapture_sbom/evidence.py | 88 +--- scripts/sbom/isaaccapture_sbom/inventory.py | 186 ++------ scripts/sbom/isaaccapture_sbom/licensing.py | 362 +++++---------- scripts/sbom/isaaccapture_sbom/report.py | 74 +-- scripts/sbom/isaaccapture_sbom/validate.py | 134 ++---- scripts/sbom/isaaccapture_sbom/wheelfile.py | 59 +-- scripts/sbom/pyproject.toml | 16 +- scripts/sbom/tests/conftest.py | 25 +- scripts/sbom/tests/synth.py | 96 +--- scripts/sbom/tests/test_discovery.py | 99 +--- scripts/sbom/tests/test_elf.py | 3 +- scripts/sbom/tests/test_gates.py | 87 +--- scripts/sbom/tests/test_licensing.py | 68 +-- scripts/sbom/tests/test_nested_terms.py | 5 +- scripts/sbom/tests/test_spdx_conformance.py | 3 +- scripts/sbom/tests/test_static_archives.py | 14 +- scripts/sbom/tests/test_wheel_sbom.py | 57 +-- src/core/python/CMakeLists.txt | 13 +- src/core/python/generate_stubs.py | 13 +- src/core/python/license_files.py | 2 +- src/core/python/pyproject.toml.in | 4 +- tests/python/packaging/test_wheel_contents.py | 1 - 38 files changed, 566 insertions(+), 1669 deletions(-) diff --git a/.github/workflows/build-ubuntu.yml b/.github/workflows/build-ubuntu.yml index b996f23b5d..c812cde7b8 100644 --- a/.github/workflows/build-ubuntu.yml +++ b/.github/workflows/build-ubuntu.yml @@ -284,10 +284,7 @@ jobs: mv install/wheels-repaired/*.whl install/wheels/ rmdir install/wheels-repaired || true - # Runs between auditwheel repair and every upload, because the document - # describes the finished wheel and the wheel gains files here. - # The inventory is discovered from this build tree -- CMake's file API - # codemodel, the fetched sources -- so nothing has to be declared anywhere. + # Run after auditwheel and before upload so evidence describes the repaired bytes. - name: Package licenses and contents SBOM into wheels (Release only) if: matrix.build_type == 'Release' run: | @@ -315,7 +312,6 @@ jobs: --manifest sbom/manifest.json \ --build-evidence "sbom/${stem}.build-evidence.json" done - # Third-party conformance, from the collector's own dev pins. for document in sbom/*.spdx.json; do uv run --project scripts/sbom --python ${{ matrix.python_version }} \ --extra dev pyspdxtools -i "${document}" @@ -327,15 +323,12 @@ jobs: set -euo pipefail shopt -s nullglob wheels=(install/wheels/*.whl) - # uv pip install re-checks every RECORD digest, so this doubles as proof - # that the rewrite left an installable wheel behind. --find-links resolves - # the isaacteleop transition wheel built alongside it; without it uv goes - # to PyPI for a dependency this build just produced. + # --find-links resolves the sibling transition wheel from this build. uv venv --python ${{ matrix.python_version }} .venv-wheel-check uv pip install --python .venv-wheel-check \ --find-links install/wheels "${wheels[@]}" .venv-wheel-check/bin/python -c "import isaaccapture; print('imported', isaaccapture.__version__)" - # The consumer-side check, run exactly as the published instructions do. + # Exercise standalone verification without build evidence or a manifest. for wheel in "${wheels[@]}"; do uv run --project scripts/sbom --python ${{ matrix.python_version }} \ isaaccapture-sbom check --wheel "${wheel}" @@ -1155,9 +1148,7 @@ jobs: echo "All wheel dependency jobs completed successfully" publish-sbom-evidence: # durable copies attached to the GitHub release - # Actions artifacts expire in days; a tagged release is where the evidence a - # consumer is told to fetch actually lives. Wheels keep their own copy - # embedded either way, so this job publishes, it does not gate. + # Attach durable evidence to releases; Actions artifacts expire. runs-on: ubuntu-22.04 needs: - wheel-gate @@ -1167,9 +1158,7 @@ jobs: env: ARCHIVE: isaaccapture-sbom-${{ github.ref_name }} - # `!cancelled()` for the reason spelled out on `publish-wheel`: a skip - # anywhere upstream of `wheel-gate` reaches here too, and the result check - # is what should decide. + # !cancelled() lets wheel-gate's result decide despite skipped upstream jobs. if: >- !cancelled() && needs.wheel-gate.result == 'success' @@ -1199,8 +1188,7 @@ jobs: pattern: isaaccapture-sbom-* path: sbom-sidecars - # The merged manifest and the archive are stamped with this, so re-running - # this job publishes the same bytes. + # Fix manifest and archive timestamps for reproducible reruns. - name: Pin the build clock to the commit run: echo "SOURCE_DATE_EPOCH=$(git log -1 --pretty=%ct)" >> "$GITHUB_ENV" @@ -1208,26 +1196,23 @@ jobs: run: | set -euo pipefail - # A `py3-none-any` wheel arrives from every matrix entry; the merge - # refuses one whose copies disagree, and copies only the evidence the - # merged manifest binds. + # Universal wheels repeat across matrix entries. Reject conflicting copies + # and publish only the evidence bound by the merged manifest. uv run --project scripts/sbom --python 3.12 \ isaaccapture-sbom merge-manifests \ --output "${ARCHIVE}/isaaccapture-sbom-manifest.json" \ --evidence-out "${ARCHIVE}" \ sbom-sidecars/*/manifest.json - # One zip, not loose assets: GitHub renames release assets (`!` becomes - # `.`), and names inside a zip are kept. Fixed times, modes and order, - # and no owner or extra time fields (-X), so a rerun zips the same - # bytes; zip stores local time, hence TZ. + # ZIP preserves filenames that GitHub renames in loose assets (`!` to `.`). + # Fix times, modes, order, and extra fields for reproducibility; + # ZIP stores local time, so set TZ too. touch -d "@${SOURCE_DATE_EPOCH}" "${ARCHIVE}"/* chmod 644 "${ARCHIVE}"/* printf '%s\n' "${ARCHIVE}"/* | LC_ALL=C sort \ | TZ=UTC zip -X -q "${ARCHIVE}.zip" -@ - # Rehearses the published verification instructions on what a consumer - # unpacks, and re-checks every advertised digest. + # Verify the unpacked release using the published consumer procedure. unzip -q "${ARCHIVE}.zip" -d unpacked uv run --project scripts/sbom --python 3.12 \ isaaccapture-sbom check-set \ @@ -1243,9 +1228,8 @@ jobs: run: | set -euo pipefail - # release-tag.yml pushes the tag before it creates the draft release, so - # this job can start first. Wait for the release rather than creating one, - # which would bypass the draft/approval step. + # The tag can trigger this job before release-tag.yml creates its draft. + # Wait for that release to preserve its draft/approval step. deadline=$(( SECONDS + 900 )) until gh release view "${TAG}" >/dev/null 2>&1; do if (( SECONDS >= deadline )); then @@ -1256,8 +1240,6 @@ jobs: sleep 20 done - # --clobber makes a retry idempotent. Draft state and release notes are - # left exactly as they were. gh release upload "${TAG}" "${ARCHIVE}.zip" --clobber echo "Attached ${ARCHIVE}.zip to ${TAG}" @@ -1312,13 +1294,11 @@ jobs: pattern: isaaccapture-sbom-* path: sbom-sidecars - # Nothing is published until every variant the matrix produces is here and - # still hashes to what its own manifest recorded. - name: Verify wheel variant coverage and digests run: | set -euo pipefail - # Mirrors the build-ubuntu Release matrix: 2 architectures x 3 Pythons. + # Keep platform and ABI lists aligned with the build-ubuntu Release matrix. platforms=(x86_64 aarch64) abis=(cp311 cp312 cp313) expected=$(( ${#platforms[@]} * ${#abis[@]} )) @@ -1350,8 +1330,7 @@ jobs: done < <(jq -r '.wheels[] | [.filename, .sha256] | @tsv' "${manifest}") done - # Every wheel under wheels/ is uploaded below, so one no manifest names - # would be published with no evidence checked for it. + # Every downloaded wheel is uploaded below, so require evidence for all of them. for wheel in wheels/*.whl; do name="$(basename "${wheel}")" if [[ -z "${named[${name}]:-}" ]]; then diff --git a/.gitignore b/.gitignore index c7ce6a078e..1355488147 100644 --- a/.gitignore +++ b/.gitignore @@ -70,7 +70,7 @@ MUJOCO_LOG.TXT # Runtime device files injected by MCP server infrastructure .mcp.json -# Coverage databases, written beside whichever suite was measured +# Python coverage databases .coverage .coverage.* diff --git a/CMakeLists.txt b/CMakeLists.txt index e666d0be98..ded7797a6b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -3,11 +3,9 @@ cmake_minimum_required(VERSION 3.24...3.25) -# The SBOM collector reads this build's link and compile graph, and the -# directories the compiler hands the linker, from the file API reply. The query -# is feature-tested rather than required: cmake_file_api() arrived in 3.27, and -# raising the minimum opts into CMP0148, which removes the FindPythonInterp -# module pybind11's legacy path still calls through vcpkg's find_package wrapper. +# SBOM discovery needs the codemodel and toolchains replies (CMake >=3.27). +# Feature-test the query: raising the policy version would enable CMP0148, +# removing FindPythonInterp still used by pybind11's legacy path through vcpkg. if(COMMAND cmake_file_api) cmake_file_api(QUERY API_VERSION 1 CODEMODEL 2 TOOLCHAINS 1) endif() diff --git a/deps/cloudxr/.gitignore b/deps/cloudxr/.gitignore index 5897b633ff..17b5d1e62a 100644 --- a/deps/cloudxr/.gitignore +++ b/deps/cloudxr/.gitignore @@ -1,6 +1,6 @@ # CloudXR runtime and SDK tarballs *.tar.gz -# Where the download came from, written beside it by the fetch script. +# Download provenance sidecars *.tar.gz.source *.tgz diff --git a/deps/cloudxr/openxr_extensions/CMakeLists.txt b/deps/cloudxr/openxr_extensions/CMakeLists.txt index aedea5d8d7..eb0e6412f6 100644 --- a/deps/cloudxr/openxr_extensions/CMakeLists.txt +++ b/deps/cloudxr/openxr_extensions/CMakeLists.txt @@ -1,9 +1,7 @@ # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -# NVIDIA's own OpenXR extension headers. The ones written by Khronos and -# Collabora live in deps/third_party/openxr_extensions; this target carries them -# so that linking Teleop::openxr_extensions still reaches every extension header. +# Carry the upstream headers through this target to preserve the public interface. add_library(teleop_openxr_extensions INTERFACE) add_library(Teleop::openxr_extensions ALIAS teleop_openxr_extensions) diff --git a/deps/third_party/CMakeLists.txt b/deps/third_party/CMakeLists.txt index 15dc783a0a..76352a70c6 100644 --- a/deps/third_party/CMakeLists.txt +++ b/deps/third_party/CMakeLists.txt @@ -285,10 +285,8 @@ endif() # ============================================================================== # SPDX license list data # ============================================================================== -# Reference license texts. The SBOM collector names a dependency's license by -# matching the text that dependency ships against these; nothing from this -# checkout is redistributed, and it builds nothing. Pinned by commit, so no tag -# upstream can move underneath a build. +# Reference texts for license matching; this checkout builds and ships nothing. +# Pin by commit because upstream tags can move. find_package(Git REQUIRED) set(SPDX_LICENSE_DATA_COMMIT "31ba1a50e5397e00a304dbadc76531740e89ee48") # v3.29.0 diff --git a/deps/third_party/fetch_license_list_data.cmake b/deps/third_party/fetch_license_list_data.cmake index 0fcb3eb6e2..e9bdcd1e3a 100644 --- a/deps/third_party/fetch_license_list_data.cmake +++ b/deps/third_party/fetch_license_list_data.cmake @@ -1,8 +1,7 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -# Narrowed to the JSON the SBOM collector reads: upstream publishes every license -# in several formats, none of which anything here consumes. +# Fetch only the JSON corpus used by the SBOM collector. foreach(_required GIT_EXECUTABLE SPDX_REPOSITORY SPDX_COMMIT SPDX_SOURCE_DIR SPDX_SPARSE_DIR) if(NOT DEFINED ${_required}) @@ -19,9 +18,8 @@ if(EXISTS "${SPDX_SOURCE_DIR}/.git") RESULT_VARIABLE _rev_parse_result ) if(_rev_parse_result EQUAL 0 AND _current_commit STREQUAL SPDX_COMMIT) - # Same commit, but narrowed differently. SPDX_SPARSE_DIR is part of the - # download command, so changing it re-runs this. A directory, not files: - # cone mode refuses a file path once the checkout holds it. + # The download command can change the sparse path without changing the commit. + # Cone mode requires a directory, not a file path. execute_process( COMMAND "${GIT_EXECUTABLE}" sparse-checkout set "${SPDX_SPARSE_DIR}" WORKING_DIRECTORY "${SPDX_SOURCE_DIR}" diff --git a/deps/third_party/openxr_extensions/CMakeLists.txt b/deps/third_party/openxr_extensions/CMakeLists.txt index 3065fa186b..45376d71c8 100644 --- a/deps/third_party/openxr_extensions/CMakeLists.txt +++ b/deps/third_party/openxr_extensions/CMakeLists.txt @@ -1,14 +1,7 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -# OpenXR extension headers written by other people, vendored here verbatim: -# -# openxr_extension_helpers.h The Khronos Group Inc. Apache-2.0 -# XR_MNDX_xdev_space.h Collabora, Ltd. BSL-1.0 -# -# Kept apart from deps/cloudxr/openxr_extensions, which holds NVIDIA's own, so -# that what this project redistributes under someone else's terms is visible -# from the directory layout rather than only from the file headers. +# Keep upstream extension headers separate from NVIDIA headers for license attribution. add_library(teleop_openxr_extensions_upstream INTERFACE) add_library(Teleop::openxr_extensions_upstream ALIAS teleop_openxr_extensions_upstream) diff --git a/pyproject.toml b/pyproject.toml index 88dd88a8f4..fca4fb6dde 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -15,9 +15,8 @@ description = "Isaac Teleop" readme = "README.md" requires-python = ">=3.11" license = "Apache-2.0" -# The license texts every isaaccapture wheel ships; src/core/python/CMakeLists.txt -# reads this list for the setuptools wheel, so name a text here and nowhere else. -# Not a glob over LICENSES/: that pool also licenses components no wheel carries. +# Shared with the setuptools wheel via src/core/python/CMakeLists.txt. +# Avoid a LICENSES/ glob: the pool includes terms for components no wheel carries. license-files = [ "LICENSE.md", "LICENSES/Apache-2.0.txt", @@ -117,10 +116,8 @@ isaacteleop = { path = "src/compat" } [tool.scikit-build] # EgoPost is hosted here but is not part of the Isaac Capture distribution. sdist.exclude = ["src/python/isaaccapture_egopost/**", "LICENSES/LicenseRef-OpenMDW-1.1.txt"] -# Floor matches CMakeLists' cmake_minimum_required (3.27, for cmake_file_api). -# Ubuntu 22.04's apt CMake is 3.22 and no longer satisfies it, so scikit-build-core -# fetches CMake from PyPI there; the <4 cap keeps that fetch on the 3.x line rather -# than silently jumping to 4.x. +# SBOM file-API queries require CMake 3.27. scikit-build-core fetches a newer +# CMake when the system copy is too old; keep that fetch on the 3.x line. cmake.version = ">=3.27,<4" cmake.build-type = "Release" diff --git a/scripts/download_cloudxr_runtime_sdk.sh b/scripts/download_cloudxr_runtime_sdk.sh index d45e91649c..0b1b7f5a80 100755 --- a/scripts/download_cloudxr_runtime_sdk.sh +++ b/scripts/download_cloudxr_runtime_sdk.sh @@ -112,10 +112,7 @@ download_ngc_file() { rm -f "$out_path" return 1 fi - # Record where this came from. The tarball is fetched, not checked in, so - # without this the only evidence of its origin is a shell variable that - # stops existing when the script exits -- and the SBOM has to call the - # download location unknown. + # Preserve the download URL as SBOM provenance. printf '%s\n' "$url" > "${out_path}.source" } diff --git a/scripts/sbom/isaaccapture_sbom/__init__.py b/scripts/sbom/isaaccapture_sbom/__init__.py index 8f5e4361e8..11895f10c7 100644 --- a/scripts/sbom/isaaccapture_sbom/__init__.py +++ b/scripts/sbom/isaaccapture_sbom/__init__.py @@ -10,11 +10,7 @@ def stamped_now(): - """The time to publish, honouring SOURCE_DATE_EPOCH. - - Every artifact a release advertises uses this; the manifest binds them as one - chain, so they must agree on the clock. - """ + """Return the publication time, honoring SOURCE_DATE_EPOCH.""" import os from datetime import datetime, timezone diff --git a/scripts/sbom/isaaccapture_sbom/build.py b/scripts/sbom/isaaccapture_sbom/build.py index d034671432..64ef28d2b5 100644 --- a/scripts/sbom/isaaccapture_sbom/build.py +++ b/scripts/sbom/isaaccapture_sbom/build.py @@ -1,7 +1,7 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -"""Turn a repaired wheel plus its build tree into a wheel that carries both.""" +"""Package a wheel's contents SBOM and license evidence from its build tree.""" from __future__ import annotations @@ -37,15 +37,11 @@ from .inventory import Attribution from .wheelfile import Entry -# Only the namespace this tool mints documents under. Supplier, licence and -# homepage are read from the wheel's own packaging metadata, so a fork, a rename -# or a second distribution built from the same tree cannot leave this file -# stating facts about somebody else's. +# Only the document namespace is fixed; distribution identity comes from metadata. PROJECT = {"document_namespace": "https://github.com/NVIDIA/IsaacCapture/spdx"} def _project_facts(repo_root: Path, metadata: dict) -> dict: - """What this distribution says about itself, from its own metadata.""" urls = metadata.get("project_urls") or {} homepage = next( ( @@ -55,21 +51,17 @@ def _project_facts(repo_root: Path, metadata: dict) -> dict: ), metadata.get("home_page") or NOASSERTION, ) - # The wheel's own `Author` first: a second distribution built from this tree - # is a different project, and the checkout's `pyproject.toml` describes only - # the one it is the root of. + # A sibling distribution may have a different author than the root project. author = (metadata.get("author") or "").strip() return { **PROJECT, "supplier": f"Organization: {author}" if author else project_supplier(repo_root), - # `license` is the legacy free-text field; only an expression is an - # expression, and a wheel declaring neither declares nothing. + # The legacy License field is free text, not an SPDX expression. "license": metadata.get("license_expression") or NOASSERTION, "homepage": homepage, - # What the project states about itself, for a distribution whose own - # files state nothing -- a metadata-only wheel has no file to carry it. + # Metadata-only wheels have no source file carrying the project notice. "copyright": project_copyright(repo_root), } @@ -78,7 +70,6 @@ class BuildError(Exception): """The wheel cannot be published with the evidence available.""" -# Each evidence-gap error ends with this; the README section holds the remedies. _REMEDIES = "\nSee 'What stops the Release build' in scripts/sbom/README.md." @@ -97,16 +88,10 @@ def default_license_data(build_dir: Path) -> Path: def _with_reached_nested(inventory, components: dict) -> dict: - """Keep nested terms only where what they cover reached this wheel. - - A nested LICENSE states a holder for part of a component -- a bundled - library, an asset set -- and dropping it names the wrong holder for those - bytes. Keeping all of them names holders for code that never shipped. - """ - # One entry per path and how, under its first member: members linking one - # library read the same headers, and every nested text scans these. + """Keep nested terms only for files or compiled code that reached this wheel.""" + # Deduplicate reach shared by several members, retaining the first member as evidence. reached: dict[str, dict[tuple[str, str], Reach]] = {} - # A linked archive of the component holds what it holds: all of it reached. + # A prebuilt archive conservatively retains all nested terms of its component. prebuilt: dict[str, str] = {} for member, attribution in inventory.attributions.items(): for key, paths in attribution.read.items(): @@ -156,36 +141,29 @@ def _declared_without_text(declared: str, shipped: set[str]) -> list[str]: try: identifiers = licensing.identifiers_in(declared) except licensing.ExpressionReadError: - # An expression nothing can read names nothing that can be checked, and - # the document carries it verbatim; say so rather than pass silently. + # Malformed declarations must fail the missing-text gate. return [declared] return sorted( item for item in identifiers - # An exception modifies the licence it is attached to and has no text in - # the list this build fetches, so demanding one blocks every - # `X WITH ` declaration with an instruction nothing can meet. + # Exceptions modify a license; this gate requires texts only for license operands. if not licensing.is_exception(item) and f"{item.lower()}.txt" not in shipped ) def _own_texts(shipped: set[str]) -> set[str]: - """The texts a wheel carries as its own, not as a third party's. + """Return paths relative to .dist-info/licenses/, excluding third-party terms. - These are named relative to `.dist-info/licenses/`, so a third party's sits - under `third-party/`. Counting those let somebody else's copy discharge this - distribution's obligation -- remove that component and the wheel silently - loses its own terms. + A component's text must not discharge the distribution's own obligation: removing + that component would also remove the distribution's license. """ return {item for item in shipped if not item.startswith("third-party/")} def _own_license_texts(declared: str, shipped: set[str]) -> dict[str, str]: - """Reference texts for the ids this distribution declares and does not ship. + """Supply reference texts missing from the distribution's declared licenses. - The identifiers come from the wheel's own `License-Expression`, so this adds - the terms the distribution says apply to itself -- never a third party's, - which are packaged from the files those components actually carry. + Third-party terms must instead come from the components' own files. """ if declared == NOASSERTION: return {} @@ -199,12 +177,9 @@ def _own_license_texts(declared: str, shipped: set[str]) -> dict[str, str]: def siblings_of(wheels: list[Path], repo_root: Path) -> dict[str, dict]: - """What each wheel of this run states about itself, by canonical name. + """Read sibling wheel metadata before rendering, indexed by canonical name. - Read before any of them is described, so a requirement naming one is never - reported as an install-time unknown. Two wheels can require each other -- a - renamed project and its transition wheel do -- and both can state what the - metadata says, because that needs no document. + Metadata supports cyclic requirements without cyclic document hashes. """ known: dict[str, dict] = {} for wheel in wheels: @@ -250,16 +225,8 @@ def build( wheel, metadata_raw, resolver, discovery.staged_root ) - # Two settlements before anything renders, applied to both views of a - # component so the document, the reports, the packaged notices and the - # evidence sidecar all describe the same thing. - # - # Terms stated for one subdirectory apply only if something under it - # reached this wheel: a test suite's licence is not this wheel's business, - # a vendored asset set's or a compiled-in library's is. And components - # carrying byte-identical terms share one LicenseRef -- settled here because - # deciding it in the renderer left the notices file naming an id the SBOM - # beside it did not define. + # Resolve nested terms and shared LicenseRefs before rendering any output, + # so notices, reports, and evidence use the same terms and identifiers. settled = _with_reached_nested(inventory, inventory.components_present) present = {**inventory.components_present, **settled} renamed = licensing.shared_refs(present.values()) @@ -322,8 +289,7 @@ def build( "its README; obtain the terms before publishing it." + _REMEDIES ) - # Everything the wheel gains, computed before a byte is written so the - # document can describe the wheel it is about to be embedded in. + # Compute the final member set before rendering its SBOM. additions: dict[str, bytes] = {} packaged_paths: list[str] = [] for key, component in inventory.components_present.items(): @@ -336,10 +302,7 @@ def build( report_module.declared_license_path(wheel.dist_info, target) ) - # The distribution's own declared licence is an obligation like any other. - # Apache-2.0 asks that recipients get a copy, and the gate above covers only - # what the build redistributes -- so a wheel can declare a licence whose - # text it does not carry, as a metadata-only wheel does. + # Metadata-only wheels still need their own declared license texts. declared = _project_facts(repo_root, wheel_metadata)["license"] shipped = { report_module.declared_license_path(wheel.dist_info, item.name) @@ -367,8 +330,6 @@ def build( report_module.declared_license_path(wheel.dist_info, notices_path) ) - # Whatever could not be supplied from the corpus -- a LicenseRef, an id the - # pinned list does not carry -- is a declaration with no terms behind it. unmet = _declared_without_text( declared, {Path(item).name.lower() for item in _own_texts(set(packaged_paths) | shipped)}, @@ -386,11 +347,8 @@ def build( metadata_declared = patched_metadata != metadata_raw replacements = {f"{wheel.dist_info}/METADATA": patched_metadata} - # Neither file can state its own digest. RECORD hashes every member, this - # document included, and the document is rendered before RECORD is - # regenerated -- listing either is a fixed point no hash function has. SPDX - # 2.3 requires a checksum on every file listed, so they are omitted rather - # than listed without one; `check` reads both against the archive instead. + # Exclude the SBOM and RECORD to avoid circular digests. SPDX files require + # checksums; the verifier checks these excluded members against the archive. excluded = [f"{wheel.dist_info}/RECORD", sbom_name] projected = [item for item in wheel.entries if item.name not in replacements] + [ @@ -398,16 +356,11 @@ def build( ] projected_wheel = replace(wheel, entries=tuple(projected)) for name in [*additions, *replacements]: - # Only the third-party texts: the notices file and the document below - # are this collector's own output, and reading them back would take a - # `- Copyright:` bullet of its own summary for a notice. + # Do not scrape our generated summaries as upstream copyright notices. packaged_text = name in additions and report_module.is_third_party_license( wheel.dist_info, name ) body = additions.get(name, b"").decode("utf-8", "replace") - # A licence text can state its own terms and carry a notice. The - # collector packages these itself, so it reads them here, the way it - # reads every member it describes. inventory.attributions[name] = Attribution( path=name, origin="metadata", @@ -416,17 +369,14 @@ def build( if name in additions else "distribution metadata, updated to declare the packaged license files", spdx_tag=licensing.normalized_tag(body) if packaged_text else None, - # Every holder the text names, as the package roll-up does: keeping - # the first makes a per-file notice that does not discharge the - # obligation -- libbsd's copyright file names forty-six. + # Retain all holders, including those in multi-component copyright files. copyright_text="\n".join( licensing.fold_notices(licensing.read_notices(body)) ) if packaged_text else None, ) - # The document is added below, after it has been rendered; record it now so - # the evidence accounts for every member of the finished wheel. + # Record the SBOM before rendering so the evidence covers the final member set. inventory.attributions[sbom_name] = Attribution( path=sbom_name, origin="metadata", @@ -485,8 +435,7 @@ def build( json.dumps(evidence_doc, indent=2, sort_keys=True) + "\n", encoding="utf-8" ) - # The wheel digest is bound here, after the last mutation: a wheel that is - # touched again no longer matches its own manifest entry. + # Bind the wheel digest only after the final mutation. return { "schema": "isaaccapture-sbom-manifest/1", "generated_at": stamped_now().strftime("%Y-%m-%dT%H:%M:%SZ"), @@ -512,11 +461,7 @@ def build( def merge_manifests(paths: list[Path], evidence_out: Path | None = None) -> dict: - """Fold per-variant manifests into the one a release advertises. - - With `evidence_out`, copy there the sidecars it binds, from beside the - manifest that named them, and nothing else. - """ + """Merge variant manifests, optionally copying exactly their bound sidecars.""" found = [ (path.parent, item) for path in sorted(paths) @@ -526,11 +471,10 @@ def merge_manifests(paths: list[Path], evidence_out: Path | None = None) -> dict if evidence_out is None: return merged - # From beside the very record the merge bound, so the copy cannot follow a - # different "first" than the binding did. + # Copy from the selected record's directory, not another build of the same wheel. beside = {id(item): parent for parent, item in found} bound = [(beside[id(record)], record) for record in merged["wheels"]] - # Checked before anything is written, so a refused set copies nothing. + # Validate the entire set before copying any files. failures = [ f"{record['filename']}: {failure}" for parent, record in bound @@ -540,7 +484,6 @@ def merge_manifests(paths: list[Path], evidence_out: Path | None = None) -> dict raise BuildError("; ".join(failures)) evidence_out.mkdir(parents=True, exist_ok=True) for parent, record in bound: - # Already in place, and verified above; copying onto itself would raise. if parent.resolve() == evidence_out.resolve(): continue for name in validate_module.bound_names([record]): @@ -549,12 +492,8 @@ def merge_manifests(paths: list[Path], evidence_out: Path | None = None) -> dict def merge_records(wheels: list[dict]) -> dict: - """The manifest for a set of wheels, stamped when the set was complete.""" - # A `py3-none-any` wheel is built by every matrix entry, so a release set - # names it once per entry. The wheel, and whatever is derived from it alone, - # must agree, or one filename stands for things a consumer cannot tell apart. - # Build evidence describes the build, and every entry built this wheel, so - # the first entry's is the one the manifest binds. + # Universal wheels repeat across matrix entries. Their bytes and wheel-derived + # sidecars must agree; build evidence may differ, so bind the first entry's. identity = ("filename", "sha256", "size", "sbom_in_wheel") unique: dict[str, dict] = {} conflicting: set[str] = set() @@ -590,7 +529,7 @@ def merge_records(wheels: list[dict]) -> dict: def environment_summary() -> dict: - """Recorded so a rerun can tell a cold build from a cache-restored one.""" + """Record CI/cache context for reproducibility.""" return { "runner_os": os.environ.get("RUNNER_OS"), "github_run_id": os.environ.get("GITHUB_RUN_ID"), diff --git a/scripts/sbom/isaaccapture_sbom/cli.py b/scripts/sbom/isaaccapture_sbom/cli.py index 927deafe66..84fdf82caa 100644 --- a/scripts/sbom/isaaccapture_sbom/cli.py +++ b/scripts/sbom/isaaccapture_sbom/cli.py @@ -22,10 +22,9 @@ def _repo_root(value: str | None) -> Path: - """The checkout to inventory: given, else the one the caller is standing in. + """Resolve the requested checkout, defaulting to the caller's working tree. - Not derived from this file's location -- the collector is installed into an - environment of its own, so its own path says nothing about the checkout. + The installed collector's location is unrelated to the source checkout. """ if value: return Path(value).resolve() @@ -73,8 +72,6 @@ def _cmd_build(args: argparse.Namespace) -> int: manifests.append(manifest) print(f"packaged evidence into {wheel.name}") - # The merge a release uses: the set is stamped after its last wheel, and one - # filename cannot stand for two wheels. merged = build_module.merge_records( [item for manifest in manifests for item in manifest["wheels"]] ) @@ -216,17 +213,10 @@ def main(argv: list[str] | None = None) -> int: licensing.ExpressionReadError, discovery.ArchiveError, discovery.SupplierError, - # Raised both for a build tree with no file API reply and for a repo - # root git cannot answer for; its message is the instruction to the - # operator, and a traceback buries it. discovery.FileApiError, wheelfile.WheelError, validate_module.ManifestError, - # Malformed input reaches these through a document, a manifest or a - # wheel's own metadata: a corrupt JSON body, a requirement that will not - # parse, a `--wheel` that is not a zip, a manifest missing a key, a file - # that is not there. Each has something worth printing; a traceback - # buries it and exits 1 where the caller expects 2. + # Report malformed inputs as CLI errors (exit 2), without a traceback. ValueError, OSError, KeyError, diff --git a/scripts/sbom/isaaccapture_sbom/discovery.py b/scripts/sbom/isaaccapture_sbom/discovery.py index 8ede6ece7a..f8fd6648ab 100644 --- a/scripts/sbom/isaaccapture_sbom/discovery.py +++ b/scripts/sbom/isaaccapture_sbom/discovery.py @@ -1,18 +1,10 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -"""Find out what went into a build by reading what the build left behind. +"""Discover components and provenance from build outputs. -There is no dependency list to keep in step with the code. A component exists -because a source tree was fetched into the build, and it reaches a wheel because -the link graph or a content hash says so. - -Four indexes do the work: - -* source trees ``_deps/*-src`` trees -> identity, from git or the declaration -* build graph CMake's file API codemodel -> what links and compiles into what -* archives members of tarballs in the repo, by content hash -* files repo and build-tree files, by content hash +Indexes cover fetched trees, the CMake graph, archive members, and repository +and generated files. See scripts/sbom/README.md for attribution rules. """ from __future__ import annotations @@ -39,7 +31,6 @@ from . import elf, licensing -# Directories that never hold an input worth indexing. _SKIP_DIRS = { ".git", ".venv", @@ -51,15 +42,12 @@ } ARCHIVE_SUFFIXES = (".tar.gz", ".tgz", ".tar.xz", ".tar.bz2", ".tar", ".zip") _MAX_ARCHIVE_BYTES = 1 << 31 -# How much of a library nested in an archive is kept, to read its build-id from. -# An SDK ships its libraries under `lib/`, not at the archive root, so without -# this the only members with a build-id are the ones nothing links against. +# Retain ELF prefixes even for nested archive members to read their build IDs. _ELF_HEAD_BYTES = 1 << 16 _GITHUB_REMOTE = re.compile( r"github\.com[:/](?P[^/]+)/(?P[^/]+?)(?:\.git)?/?$" ) -# A release asset or a source archive on GitHub names its repository and its ref. _GITHUB_REPO = r"(?P[^/]+)/(?P[^/]+)/" _GITHUB_DOWNLOADS = ( re.compile( @@ -74,9 +62,8 @@ r"(?:refs/(?:tags|heads)/)?(?P.+)$" ), ) -# Schemes this collector states as a downloadLocation; a local path names only the builder. +# Local download paths identify only the builder, not a public origin. _DOWNLOAD_SCHEMES = ("http", "https", "ftp", "ftps") -# Transports a published git location may name; a local path names only the builder. _GIT_TRANSPORTS = ("http", "https", "ssh", "git") # git's scp-like `[user@]host:path`, which SPDX knows only as `ssh://`. _SCP_REMOTE = re.compile(r"^(?P[^@/:]+@)?(?P[^@/:]{2,}):(?!//)/?") @@ -92,12 +79,7 @@ def component_of(display: str) -> str: - """The component a `FileIndex` display path belongs to. - - `add_tree` renders `/` and a fetched checkout's tree is - `-src`, so this is the inverse of how the index was populated. Inverting - it at the call site made the convention an undeclared contract. - """ + """Recover the component key from FileIndex's -src/ display path.""" return display.split("/", 1)[0].removesuffix("-src") @@ -128,9 +110,7 @@ def _walk(root: Path): yield Path(entry.path) -# ============================================================================== # Source trees -# ============================================================================== @dataclass(frozen=True) @@ -150,23 +130,18 @@ class Component: root: Path | None = None license_concluded: str = "NOASSERTION" license_declared: str = "NOASSERTION" - # The component's own copyright notice, where the build can read one. Naming - # a holder and then declaring none would contradict the same document. copyright_text: str = "NOASSERTION" evidence: tuple[licensing.LicenseEvidence, ...] = () - # (path, reason) for checked-in files whose REUSE info fails the per-file - # gate. Pooling a holder's licences would otherwise lend one file the terms - # of another, or void them all. + # (path, reason) for per-file REUSE failures; a holder's other files cannot + # lend their licenses to an unlicensed file. unlicensed: tuple[tuple[str, str], ...] = () def own_git(directory: Path, *args: str) -> str | None: - """A read-only git query about the checkout rooted at `directory`. + """Query git only when directory is the checkout root. - git answers for whichever repository encloses a directory, and `build/` sits - inside this project's checkout, so a tree fetched as an archive would - otherwise be identified as this project. None unless git's toplevel is - `directory` itself. + An archive extracted beneath the project must not inherit the enclosing + repository's identity. """ if not _is_own_checkout(directory.resolve()): return None @@ -181,13 +156,11 @@ def _is_own_checkout(resolved: Path) -> bool: def git(repo: Path, *args: str) -> str | None: - """Run a read-only git query about `repo`, or return None if git cannot answer. + """Run a read-only git query, returning None on failure. - git exports GIT_DIR and its kin to the hooks it runs; with them set, git - answers for that repository whatever `-C` names, so they are dropped. - Configuration passed as GIT_CONFIG_PARAMETERS or GIT_CONFIG_COUNT, such as a - container's `safe.directory`, is kept, as git keeps it for a submodule; - GIT_CONFIG is not, since it replaces the repository's own configuration. + Clear repository-local variables (including GIT_DIR and GIT_CONFIG) that + override -C. Preserve GIT_CONFIG_PARAMETERS/COUNT for settings such as + safe.directory, following git's submodule behavior. """ local = set(_git_local_variables()) env = {name: value for name, value in os.environ.items() if name not in local} @@ -208,16 +181,8 @@ def version_in_name(name: str) -> str | None: def notice_from(evidence) -> str: - """The copyright a component's own licence text states, where it states one. - - Permissive licences generally require the notice to travel with the terms, - and for MIT and the BSDs it is a line of the same file; this states it - where a reader can act on it. - """ - # Grants only, though a pool counts as licence evidence everywhere else: a - # pool is a directory of reference texts, and the notice in one is the - # licence's own author -- OpenXR ships WTFPL, so reading its pool credited - # OpenXR to the person who wrote WTFPL. + """Read copyright notices from the component's license grants.""" + # Pool texts can name the license author (for example WTFPL's), not the component. notices = [ found for item in evidence @@ -240,7 +205,6 @@ class _Identity(NamedTuple): def _github_identity(org: str, repo: str, ref: str | None) -> tuple[str, str, str]: - """Supplier, homepage and purl of a GitHub repository at `ref`.""" purl = PackageURL(type="github", namespace=org, name=repo, version=ref) return f"Organization: {org}", f"https://github.com/{org}/{repo}", purl.to_string() @@ -248,8 +212,7 @@ def _github_identity(org: str, repo: str, ref: str | None) -> tuple[str, str, st def _identity( remote: str | None, name: str, commit: str | None, tag: str | None = None ) -> _Identity: - """A git source's identity, read off its remote URL, at the commit, else at - the declared tag.""" + """Identify a git source by remote URL and commit, falling back to its declared tag.""" ref = commit or tag version = ref or "NOASSERTION" match = _GITHUB_REMOTE.search(remote) if remote else None @@ -278,8 +241,7 @@ def _identity( def _archive_identity(url: str, digest: str | None, name: str) -> tuple[_Identity, str]: - """An archive's identity, read off the URL it was declared with, and how the - declaration reads in `sourceInfo`.""" + """Identify an archive from its declared URL and record its sourceInfo.""" parsed = urlparse(url) file_name = PurePosixPath(parsed.path).name downloadable = parsed.scheme.lower() in _DOWNLOAD_SCHEMES @@ -324,10 +286,7 @@ class _Declaration: def _stamps(deps_dir: Path) -> dict[str, dict[str, dict[str, str]]]: - """Each tree's FetchContent stamps, by method. - - scripts/sbom/README.md says where from. - """ + """Index FetchContent stamps by tree and download method.""" # Under the populate sub-build, or, with CMP0168 NEW, the build's own CMakeFiles. roots = ( (deps_dir, "*-subbuild/*-populate-prefix/src/*-populate-stamp"), @@ -445,9 +404,7 @@ def discover_source_trees(deps_dir: Path) -> dict[str, Component]: return components -# ============================================================================== # Build graph -# ============================================================================== @dataclass(frozen=True) @@ -472,18 +429,14 @@ class _Node: # library's own requirements land on its consumers' lines. linked: tuple[Path, ...] = () # outputs of targets in this graph inputs: tuple[LinkInput, ...] = () # everything else that can carry code - # The sources it compiles, as opposed to merely lists, and the directory - # its objects and their dependency output land in. + # Sources with compile groups; merely listed sources are not necessarily compiled. compiled: tuple[Path, ...] = () objects: Path | None = None -# How a path of a component reached an artifact, strongest evidence first. A -# member made from a file of the component, then the sources compiled into it, -# then the headers the compiler's dependency output says it read. A target that -# compiles something but left no such output stands in the sources it lists and -# its include directories. A linked archive of the component is not a path: its -# contents are unknowable, so it stands for the whole component. +# Evidence strength for nested-license reach. Missing compiler records fall +# back to listed sources and include directories. Prebuilt archives retain +# the whole component's terms because their source contents are unknown. REACH = ( "copied-file", "derived-file", @@ -496,8 +449,7 @@ class _Node: class Reach(NamedTuple): - """The evidence that kept a nested text: how, through which path, into - which wheel member.""" + """Evidence retaining a nested license: method, source path, and wheel member.""" how: str path: str @@ -554,11 +506,7 @@ class _Line: @dataclass(frozen=True) class _Search: - """Where `-l` looks after the `-L` directories, and the sysroot `=` names. - - The linker's own directories are read under `sysroot`, which a target's - CMAKE_SYSROOT replaces. - """ + """Implicit linker search directories and the sysroot used for leading = paths.""" compiler_dirs: list[Path] compiler: str | None @@ -614,12 +562,10 @@ def directories(self) -> list[Path]: def _linker_words(fragments: list[dict]) -> tuple[list[str], list[str]]: - """What a link line hands the linker, in order, and fragments that do not parse. + """Unpack linker words and retain unparseable fragments. - Every role is read, because CMake files a path given as a link option - under `flags`. Driver options go with their values; `-Wl,` lists and - `-Xlinker` words are unpacked, which is how `--whole-archive` wrappers - arrive. The driver's `-static` governs the whole line wherever it sits. + Read every CMake role: paths passed as link options appear under flags. + Unpack -Wl, and -Xlinker, and apply the driver's -static to the whole line. """ stream: list[str] = [] unparsed: list[str] = [] @@ -702,10 +648,7 @@ def _inspect(path: Path) -> tuple[str | None, int | None, list[str] | None]: def _script_inputs(path: Path) -> list[str] | None: - """The files and `-l` names a GNU ld script's INPUT and GROUP commands name. - - None if the file is not such a script. - """ + """Return INPUT/GROUP paths and -l names, or None for a non-linker-script file.""" try: with path.open("rb") as handle: head = handle.read(1 << 16) @@ -750,11 +693,7 @@ def _owner(path: Path, ownership: Ownership | None) -> str | None: def _include_owners(include: Path, ownership: Ownership | None) -> set[str]: - """The components an include directory credits. - - Which file in it was used is unknowable, so every component with files - there is credited, the same way a fetched include directory is. - """ + """Credit every owner under an include directory when individual headers are unknown.""" match = _DEPS_SOURCE.search(include.as_posix()) if match: return {match.group("name")} @@ -774,11 +713,7 @@ class FileApiError(Exception): def _within(repo_root: Path, build_dir: Path): - """Predicate: under the checkout, outside its build tree. - - Both roots are resolved by the caller; `--build-dir` is free-form and a - relative one compared against absolute paths matches nothing. - """ + """Test checkout membership outside the build tree; both roots must be resolved.""" def check(path: Path) -> bool: resolved = path if path.is_absolute() else path.resolve() @@ -790,14 +725,10 @@ def check(path: Path) -> bool: def _reply_objects(reply: Path) -> tuple[dict[str, Path], str]: - """The current reply's objects by kind, reached through its index, and the generator. - - Every other reply filename is documented as unspecified and not to be - interpreted, and CMake only *attempts* to delete what it did not just - write -- so globbing for a codemodel can find one from an earlier configure, - and sorting the matches orders content hashes. The index alone carries an - order: "the one with the largest name in lexicographic order is the current - index file". + """Read objects from the lexicographically latest index. + + Other reply filenames contain unordered content hashes, and stale codemodels + may survive reconfiguration. Do not select them by glob or filename order. """ indexes = sorted(reply.glob("index-*.json")) if reply.is_dir() else [] if not indexes: @@ -809,7 +740,6 @@ def _reply_objects(reply: Path) -> tuple[dict[str, Path], str]: for item in index.get("objects", []): version = item.get("version") or {} if item.get("kind") in wanted and wanted[item["kind"]] == version.get("major"): - # Index references are relative to the reply directory. found = reply / item["jsonFile"] if found.is_file(): objects[item["kind"]] = found @@ -834,12 +764,10 @@ def cmake_cache(build_dir: Path) -> dict[str, str]: def _search_directories(toolchains: Path | None, build_dir: Path) -> dict[str, _Search]: - """Where `-l` looks after the `-L` directories, by link language. + """Find implicit linker directories by language. - First the directories the compiler hands the linker: CMake's toolchains - reply, or the compiler asked directly for a language the reply does not - cover -- CMake records them only for languages the top-level project - enables. Then the linker's own built-in SEARCH_DIRs. + CMake records only top-level enabled languages; query other compilers directly. + Append the linker's built-in SEARCH_DIRs after the compiler's directories. """ compilers = { match["language"]: value @@ -875,7 +803,7 @@ def _search_directories(toolchains: Path | None, build_dir: Path) -> dict[str, _ def _ask(*command: str, env: dict[str, str] | None = None) -> str: - """A command's standard output, or empty where it cannot run or fails.""" + """Return command stdout, or an empty string on failure.""" try: result = subprocess.run( list(command), @@ -890,8 +818,7 @@ def _ask(*command: str, env: dict[str, str] | None = None) -> str: return result.stdout if result.returncode == 0 else "" -# A make rule as GCC and Clang write a depfile, and CMake's Makefile generators -# consolidate them into compiler_depend.make. +# GCC/Clang depfiles and CMake compiler_depend.make use make rule syntax. _MAKE_CONTINUATION = re.compile(r"\\\r?\n") _MAKE_RULE = re.compile(r"^(?:\\.|[^:\\#])+:(?:\s+(?P.*))?$") _MAKE_WORD = re.compile(r"(?:\\.|[^\s\\])+") @@ -937,16 +864,15 @@ class DependencyLog: def __init__(self, generator: str, build_root: Path) -> None: self._ninja_generator = "Ninja" in generator self._build_root = build_root - # Ninja's records grouped by the objects directory they sit in. self._ninja: dict[str, list[list[str]]] | None = None self._resolved: dict[str, Path] = {} self._read: dict[Path, tuple[set[Path], bool]] = {} def read(self, objects: Path, compiled: tuple[Path, ...]) -> tuple[set[Path], bool]: - """What compiling into `objects` read, and whether every source in - `compiled` has a record of its own: a stale object, or a generator - that keeps no record, leaves a gap, and a source another object - merely includes is not its own record.""" + """Return source/header prerequisites and whether every compiled source has its own record. + + A source included by another object does not count as its own compile record. + """ if objects not in self._read: read: set[Path] = set() recorded: set[Path] = set() @@ -1043,13 +969,9 @@ def _linker_search_dirs(compiler: str, sysroot: str) -> list[Path]: class BuildGraph: - """The link and compile graph, read from CMake's own file API. + """Read the compile/link graph from CMake's file API. - The top-level ``CMakeLists.txt`` asks for the codemodel with - ``cmake_file_api()`` where that command exists, so a configure by CMake 3.27 - or newer leaves one behind, under every generator. Scraping ``link.txt`` - instead would only work for the Makefile generators -- Ninja writes no such - file. + Do not scrape link.txt: Ninja does not produce it. """ def __init__(self, build_dir: Path, config: str | None = None) -> None: @@ -1064,7 +986,6 @@ def __init__(self, build_dir: Path, config: str | None = None) -> None: ] = {} self._dependencies = DependencyLog("", build_dir) self._linked: dict[str, tuple[_Node, ...]] = {} - # By (target name, which CMake keeps unique; the ownership asked with). self._node_reach: dict[ tuple[str, Ownership | None], dict[str, dict[Path, str]] ] = {} @@ -1095,10 +1016,7 @@ def _load(self, config: str | None) -> None: (item for item in configurations if item["name"] == (config or "")), None ) if chosen is None: - # The caller names the configuration from CMakeCache.txt, and a - # reply that does not describe it is a reply for another build -- - # taking any other credits a wheel to a graph that did not build it. - # Unnamed is the single-configuration tree the tests construct. + # Do not attribute a wheel to a different build configuration. if config: raise FileApiError( f"codemodel under {reply} describes no {config!r} configuration, " @@ -1112,8 +1030,7 @@ def resolve(raw: str, root: Path) -> Path: lines: dict[str, tuple[list[str], list[str], Path, _Search | None]] = {} for entry in chosen["targets"]: - # Relative to the codemodel, which is what the format says, even - # though CMake writes it into the reply directory alongside these. + # Target JSON paths are relative to the codemodel file. target = json.loads( (model_file.parent / entry["jsonFile"]).read_text(encoding="utf-8") ) @@ -1160,8 +1077,7 @@ def resolve(raw: str, root: Path) -> Path: self.artifacts[output] = artifact self._by_name.setdefault(output.name, []).append(artifact) - # Read once every target's outputs are known: an output of this graph - # is that target's contribution, whatever its content. + # Resolve link inputs after indexing every target output. for target_id, (words, unparsed, link_dir, found) in lines.items(): line = self._read_line(words, link_dir, found) line.inputs += [ @@ -1177,12 +1093,9 @@ def resolve(raw: str, root: Path) -> Path: def _read_line( self, words: list[str], link_dir: Path, search: _Search | None ) -> _Line: - """What a linker word stream names. + """Resolve linker inputs using all -L paths, then implicit search paths. - ld applies every `-L` to every `-l`, then the compiler's directories, - then its own built-in ones -- `search`, or None where the build records - no compiler. A `-l` name that ld would satisfy with a shared library - adds nothing, since DT_NEEDED records it. + Shared libraries are handled separately through DT_NEEDED. """ sysroot = search.sysroot if search else "" line = _Line( @@ -1224,11 +1137,7 @@ def _read_line( def _classify( self, spelling: str, path: Path, line: _Line, static: bool, depth: int = 0 ) -> None: - """Add what one path on a link line brings in, a linker script followed - under the `-Bstatic` state in force where the line names it. - - A shared library adds nothing, since DT_NEEDED records it. - """ + """Classify inputs, following linker scripts under the current -Bstatic state.""" # ld looks beside the name it was given, not beside a symlink's target. beside = path.parent path = path.resolve() @@ -1277,14 +1186,10 @@ def by_name(self, name: str) -> list[Artifact]: return self._by_name.get(name, []) def repo_inputs(self, repo_root: Path, build_dir: Path) -> set[Path]: - """Files this build compiled or included that the checkout itself holds. - - The build tree is not among them, for sources as much as for include - directories: `_deps` is the component-source domain and is attributed - from its own checkouts, generated headers state no upstream holder, and - vcpkg installs its ports under the build directory -- whose headers - carry their upstream contributors' copyright lines, which the vendored - scan reads as this repository having vendored each of those people. + """Find compiled/included files belonging to the checkout. + + Exclude the build tree: fetched sources and vcpkg headers have separate + ownership, and generated headers are not checked-in third-party sources. """ inside = _within(repo_root.resolve(), build_dir.resolve()) found: set[Path] = set() @@ -1300,12 +1205,9 @@ def repo_inputs(self, repo_root: Path, build_dir: Path) -> set[Path]: return found def _reach(self, target_id: str, prune=None) -> tuple[_Node, ...]: - """The target and everything it depends on or links, transitively. + """Traverse dependencies and linked target outputs, even without declared edges. - A link line naming another target's output brings that target, whether - or not it is also a declared dependency. A node `prune` accepts, other - than the target itself, is left out with everything reached only - through it. + Pruned nodes and paths reachable only through them are excluded; the root remains. """ seen: set[str] = set() stack = [target_id] @@ -1331,11 +1233,9 @@ class would hold every graph ever built alive for the life of the process.""" return self._closures[target_id] def static_inputs(self, artifact: Artifact) -> dict[LinkInput, set[str]]: - """What the closure links that no target here produced. + """Map external static inputs to the targets that link them. - Maps each to the targets whose link line names it. Another executable - or module is a build-order dependency -- a code generator, say -- and - nothing links it or what it links. + Exclude build-order executables/modules and their dependencies. """ if artifact.target_id not in self._static: found: dict[LinkInput, set[str]] = {} @@ -1357,13 +1257,7 @@ def _linked_closure(self, target_id: str) -> tuple[_Node, ...]: def contributions( self, artifact: Artifact, ownership: Ownership | None = None ) -> dict[str, set[str]]: - """Which components contributed to an artifact, and in what way. - - Walks the target's dependency closure. Compiled sources name a component - outright; include paths are the only trace a header-only dependency - leaves in the codemodel; a linked archive or object names whoever holds - it. - """ + """Map contributing components to compiled-source, header-include, or prebuilt roles.""" found: dict[str, set[str]] = {} def note(key: str, how: str) -> None: @@ -1449,11 +1343,7 @@ def first_party( class Ownership: - """Which component a path belongs to. - - One lookup for every kind of third-party code, so recognising a new kind is - registering a root rather than adding a branch wherever paths are matched. - """ + """Resolve ownership across registered component roots and individual files.""" def __init__(self) -> None: # path -> (owner, whether an include directory credits it outright) @@ -1461,7 +1351,7 @@ def __init__(self) -> None: self._roots: list[tuple[Path, str]] = [] # directory -> (owners credited outright, owners credited if alone) self._index: dict[Path, tuple[set[str], set[str]]] | None = None - # Every header a compile read is asked about, most of them many times. + # Cache repeated header lookups across compilation units. self._owners: dict[Path, str | None] = {} def register_root(self, root: Path, key: str) -> None: @@ -1473,11 +1363,10 @@ def roots_of(self, key: str) -> list[Path]: return [root for root, owner in self._roots if owner == key] def register_file(self, path: Path, key: str, *, by_directory: bool = True) -> None: - """Record a file's owner. + """Register a file's owner. - An include directory credits every owner of a file under it; one whose - files are registered with `by_directory=False` only where it holds every - such file there. + With by_directory=False, an include directory credits this owner only if + it owns every registered file beneath that directory. """ self._files[path.resolve()] = (key, by_directory) self._index = None @@ -1518,7 +1407,6 @@ def owners_under(self, directory: Path) -> set[str]: return found def _by_directory(self) -> dict[Path, tuple[set[str], set[str]]]: - """Each registered file's owner under the file and every directory above it.""" if self._index is None: self._index = {} for path, (key, outright) in self._files.items(): @@ -1528,16 +1416,9 @@ def _by_directory(self) -> dict[Path, tuple[set[str], set[str]]]: return self._index -# ============================================================================== -# Third-party sources vendored in this repository -# ============================================================================== -# Not everything third-party arrives through FetchContent. A file checked in -# here that someone else wrote says so in its REUSE info, which is the same -# thing the project states about its own files -- so the copyright holder is -# what separates them, not the directory it happens to sit in. +# Vendored sources: identify third parties by copyright holder, not directory. -# REUSE's tag where a file has one, and a plain copyright line where it does -# not -- upstream files usually carry the latter. +# Accept REUSE tags and plain upstream copyright notices. _COPYRIGHT_TAG = re.compile( r"SPDX-FileCopyrightText:\s*(?P[^\n\r]+)" r"|^[^\w\n]*Copyright\b(?P[^\n\r]+)", @@ -1549,11 +1430,7 @@ def _by_directory(self) -> dict[Path, tuple[set[str], set[str]]]: def project_authors(repo_root: Path) -> set[str]: - """Who the project says it is, read from its own packaging metadata. - - As written, not folded: the same names answer "is this holder ours" and - "who supplied this distribution", and only the first wants them lowercased. - """ + """Read author names verbatim; supplier output must retain their casing.""" pyproject = repo_root / "pyproject.toml" if not pyproject.is_file(): return set() @@ -1569,12 +1446,7 @@ def project_authors(repo_root: Path) -> set[str]: def project_supplier(repo_root: Path) -> str: - """Who supplied this distribution, as its own packaging metadata states it. - - SPDX takes one supplier. Choosing among several, or publishing a release - that names none, is a question for whoever is publishing it -- so it is - asked here rather than answered quietly. - """ + """Require exactly one declared author for SPDX's single supplier field.""" authors = sorted(project_authors(repo_root)) if len(authors) != 1: raise SupplierError( @@ -1586,13 +1458,10 @@ def project_supplier(repo_root: Path) -> str: def project_copyright(repo_root: Path) -> str: - """The notice this project states about itself, from its packaging metadata. + """Read the project notice from its packaging file's REUSE header. - A distribution that ships files of its own is covered by their notices. One - that ships only metadata states its copyright nowhere a reader can reach -- - the reference text its declared licence packages names no holder -- and the - REUSE header on the file the distribution is built from is where the project - does state it. + Metadata-only wheels have no source notice, and reference license texts + do not identify the distribution's copyright holder. """ pyproject = repo_root / "pyproject.toml" if not pyproject.is_file(): @@ -1619,7 +1488,6 @@ def _holder_in(text: str) -> str: def _notice_line(text: str) -> str: - # The tag introduces the notice; it is not part of it. notice = _COMMENT_MARKUP.sub("", " ".join(text.split())) return re.sub(r"^SPDX-(?:File|Snippet)CopyrightText:\s*", "", notice) @@ -1757,11 +1625,10 @@ def discover_vendored( *, tracked: Iterable[str], ) -> tuple[dict[str, Component], dict[str, list[Path]]]: - """Group checked-in files whose copyright holder is not the project's. + """Group third-party files from build inputs and REUSE-covered tracked files. - Reads the build inputs in `candidates` and the `tracked` files REUSE covers - (`_covered`); `tracked` is required, as without it tracked data and Python - go unread. Licences are REUSE's; holders are `_held_by`'s. + Tracked files include data and Python absent from the compile graph. + Licenses follow REUSE precedence; holders follow _held_by. """ # Here, not at module level: only discovery needs reuse, and reuse needs libmagic. from reuse.project import Project @@ -1803,8 +1670,7 @@ def discover_vendored( for key, paths in grouped.items(): concluded, evidence = licensing.declared_terms(key, licences.get(key, set())) declared = concluded - # What the build read first, so a shortened list names what ships; say - # when it is shortened, or it reads as the whole set. + # List build inputs first and mark truncation explicitly. named = [ name for _, name in sorted( @@ -1816,18 +1682,11 @@ def discover_vendored( ) components[key] = Component( key=key, - # Not the bare holder: an organisation in a `name` field reads as a - # product, and "Collabora Ltd" is not a package anyone can obtain. - # The holder is the only identity this material has, so it is named - # for what it is and who holds it; `sourceInfo` lists the files. name=f"Vendored source ({names[key]})", kind="vendored-source", supplier=f"Organization: {names[key]}", homepage="NOASSERTION", - # No purl: this is third-party material identified by the copyright - # holder its REUSE info or header names; there is no package to - # locate. Minting `pkg:generic/` named a package that does - # not exist, from an organisation rather than from anything shipped. + # A copyright holder is not a package identity; do not mint a purl from it. purl="NOASSERTION", version="NOASSERTION", download_location="NOASSERTION", @@ -1846,12 +1705,7 @@ def discover_vendored( return components, grouped -# ============================================================================== -# vcpkg -# ============================================================================== -# vcpkg installs into the build tree, so what it builds is neither a target of -# this build nor a fetched checkout. Its install records name the port that -# put each file there, as dpkg's do on the build host. +# vcpkg install records identify ports whose outputs are outside the CMake graph. def discover_vcpkg( @@ -1870,8 +1724,6 @@ def discover_vcpkg( port, _, rest = listing.stem.partition("_") version, _, triplet = rest.rpartition("_") key = f"vcpkg:{port}" - # A port installed for both the host and the target triplet has a list - # for each, and both installs are its files. owned.setdefault(key, []).extend( installed / line for line in listing.read_text(encoding="utf-8").splitlines() @@ -1959,12 +1811,7 @@ def _vcpkg_component( ) -# ============================================================================== -# The build host -# ============================================================================== -# The last discovery source: a shared library that no index explains, or a -# static archive linked from outside the tree, was not produced or fetched by -# this build, so ask the machine that supplied it. +# Build-host libraries: resolve inputs not explained by build or source indexes. _SEARCH_ROOTS = ( "/usr/lib/x86_64-linux-gnu", @@ -1990,12 +1837,9 @@ def _host_library_paths() -> list[Path]: @functools.lru_cache(maxsize=1) def _host_build_id_index() -> dict[str, str]: - """Host libraries by GNU build-id. + """Index host libraries by build ID, which survives SONAME/RPATH repair. - A repair tool rewrites SONAME on the way into the wheel, so the name is no - longer a way back to the machine's copy. The build-id is, and it is the same - note the linker wrote. Built once; reading it from every library the loader - knows costs about a second. + Cache the index to avoid rescanning host libraries for every wheel member. """ index: dict[str, str] = {} for path in _host_library_paths(): @@ -2012,8 +1856,7 @@ def _host_build_id_index() -> dict[str, str]: def _host_origin(library: Path | None, **known) -> dict: - """What the build machine says about a file: where it resolves, and the - package that owns it with that package's terms.""" + """Resolve a host file and identify its supplying package and terms.""" origin: dict = {**known, "resolved_path": None, "package": None, "copyright": None} if library is None: return origin @@ -2040,12 +1883,9 @@ def resolve_system_library(soname: str) -> dict: def resolve_static_archive(archive: Path) -> dict: - """Trace an archive or object the build linked to the package or toolkit it came from. + """Identify the package or toolkit supplying a linked archive or object. - A shared library is explained once its SONAME resolves on the machine: that - is how the repair tool found the file it vendored. A link line already - names a static input's path, so finding it proves nothing; only a package - or a toolkit that states its terms explains one. + The link line already supplies its path; existence alone proves no ownership. """ if not archive.is_file(): return _host_origin(None, toolkit_root=None) @@ -2090,7 +1930,6 @@ def archive_members(path: Path) -> int | None: def _identify_host_package(library: Path, origin: dict) -> dict: - """Name the package dpkg says owns a resolved host library.""" # dpkg names a multiarch package with its architecture, `libc6:i386`, and # dpkg-query -W and apt answer for one architecture only when asked so. qualified = _dpkg_owner(library) @@ -2107,7 +1946,6 @@ def _identify_host_package(library: Path, origin: dict) -> dict: return origin -# A package name as dpkg prints it, with its architecture where it has one. _DPKG_PACKAGE = re.compile(r"^[a-z0-9][a-z0-9+.-]+(?::[a-z0-9-]+)?$") @@ -2129,7 +1967,6 @@ def _dpkg_owner(library: Path) -> str | None: def _package_field(package: str, field: str) -> str | None: - """One field of a package's metadata, as dpkg records it.""" return _ask("dpkg-query", "-W", f"-f=${{{field}}}", package).strip() or None @@ -2193,11 +2030,7 @@ def _apt_origin(package: str) -> tuple[str | None, str | None]: def _package_supplier(package: str) -> str | None: - """Who ships a package dpkg can name, in the form SPDX wants. - - The package name is not a supplier: `libbsd0` names the thing, not whoever - provided it. dpkg records a maintainer, which is the answer. - """ + """Format the package Maintainer as an SPDX supplier; the package name is not one.""" maintainer = _package_field(package, "Maintainer") if not maintainer: return None @@ -2209,7 +2042,6 @@ def _package_supplier(package: str) -> str | None: def _library_roots() -> list[Path]: - """Where a pre-built library the build machine supplied can be found.""" roots = [Path(item) for item in _SEARCH_ROOTS if Path(item).is_dir()] return roots + [root / "lib64" for root in _toolkit_roots()] @@ -2234,11 +2066,9 @@ def _toolkit_root(path: Path) -> Path | None: def _license_near(library: Path) -> Path | None: - """A toolchain that installs outside the package manager still ships its terms. + """Find terms beside a library, searching up to its toolkit root. - CUDA is the case that matters: its libraries resolve into - targets//lib with EULA.txt at the toolkit root, and dpkg need know - nothing about it. Under a toolkit root the search runs up to that root. + CUDA can install outside dpkg, with targets//lib below the root EULA.txt. """ root = _toolkit_root(library) current = library.resolve().parent @@ -2256,11 +2086,9 @@ def _license_near(library: Path) -> Path | None: def _system_notice(evidence: list[licensing.LicenseEvidence]) -> str: - """The holders a machine-readable Debian copyright file names, or NOASSERTION. + """Read holders from machine-readable Debian copyright fields, or NOASSERTION. - Any other text -- an EULA, a free-form copyright file -- quotes the notices - of whatever it bundles, and scraping those names other people's code as the - holders of this library. The packaged text still carries all of it. + Do not scrape free-form agreements: bundled notices can name other components. """ for item in evidence: fields = licensing.dep5_copyright(item.text) @@ -2291,11 +2119,7 @@ def _system_purl(soname: str, origin: dict) -> str: def system_component( soname: str, origin: dict, route: str | None = None, key: str | None = None ) -> Component: - """A pre-built library the build machine supplied, named by what it came from. - - `route` says how it reached the wheel; the default is auditwheel's. `key` - defaults to `system:`. - """ + """Describe a host library; route defaults to auditwheel and key to system:.""" evidence: list[licensing.LicenseEvidence] = [] key = key or f"system:{soname}" license_path = origin.get("copyright") @@ -2308,9 +2132,7 @@ def system_component( evidence.append( licensing.evidence_for(key, "build-host", license_path, text, "grant") ) - # `key`, not `soname`: the key is what identifies this component everywhere - # else, and two components reduced to the same label would mint one LicenseRef - # for two different texts. + # Use the component key: different libraries can share a SONAME but not terms. concluded, declared = licensing.expression(evidence, key) package = origin.get("package") @@ -2337,9 +2159,7 @@ def system_component( ) -# ============================================================================== # Content indexes -# ============================================================================== @dataclass(frozen=True) @@ -2360,8 +2180,7 @@ def __init__(self) -> None: self.by_hash: dict[str, Member] = {} self.all_by_hash: dict[str, list[Member]] = {} self.by_name: dict[str, list[Member]] = {} - # Survives the RPATH/SONAME rewrite a repair tool applies on the way in, - # so a patched wheel member still points at the archive it came from. + # Build IDs survive auditwheel/patchelf RPATH and SONAME changes. self.all_by_build_id: dict[str, list[Member]] = {} self.archives: dict[str, dict] = {} @@ -2382,8 +2201,7 @@ def add_archive(self, path: Path, display: str) -> None: if Path(member.path).parent.as_posix() in {".", ""} ) for member, data in members: - # `data` is the whole member only at the archive root; elsewhere it - # is the head kept for exactly this, so ask only for the build-id. + # Nested members retain only a prefix; read their build ID, not full ELF metadata. build_id = elf.build_id_of(data) if build_id: member = replace(member, build_id=build_id) @@ -2400,8 +2218,7 @@ def _maybe_license( if Path(member.path).parent.as_posix() not in {".", ""}: return # only the archive root states the bundle's own terms if name.upper() == "VERSION" and len(data) < 256: - # First line only. These files carry more than one, and a newline in - # a version breaks every tag-value and table rendering downstream. + # Version files may contain extra lines that are not part of the version. text = data.decode("utf-8", "replace").strip().splitlines() self.archives[display]["version_text"] = text[0].strip() if text else "" return @@ -2411,19 +2228,11 @@ def _maybe_license( @staticmethod def _read_members(path: Path, display: str) -> list[tuple[Member, bytes]]: - """Hash every member; keep the bytes only of the few worth keeping. - - An SDK tarball expands to far more than it compresses to, so holding - every member would size peak memory by the largest archive rather than - by what is retained. - """ + """Stream member hashes; retain root files and ELF prefixes only.""" collected: list[tuple[Member, bytes]] = [] def take(name: str, handle) -> None: - # A root member's bytes are kept whole, for its licence text; every - # other member streams through the digest, and keeps a head only if - # it is a library, for its build-id. Reading every member whole - # sizes peak memory by the largest file in the archive. + # Retain root license texts and nested ELF prefixes without buffering large members. at_root = Path(name).parent.as_posix() in {".", ""} digest = hashlib.sha256() size = 0 @@ -2459,22 +2268,13 @@ def take(name: str, handle) -> None: if handle is not None: take(info.name.removeprefix("./"), handle) except (OSError, tarfile.TarError, zipfile.BadZipFile) as error: - # Only archives this build deliberately fetched are indexed, so one - # that will not open is a broken download, not a stray file. Saying - # so beats reporting every file it should have explained as a member - # that reached the wheel by an unknown route. + # Report unreadable fetched archives directly, before attribution failures obscure them. raise ArchiveError(f"{display} cannot be read: {error}") from error return collected def tracked_files(repo_root: Path) -> list[str]: - """What this repository actually holds, as git records it. - - A working tree also holds whatever was built in it. An install prefix has no - marker saying so, so walking the directory cannot tell the two apart, and a - wheel member matching a *build output* would be reported as a copy of - repository source. Git knows the difference. - """ + """List tracked files; a directory walk would also treat install output as source.""" listing = git(repo_root, "ls-files", "-z", "--cached") if listing is None: raise FileApiError( @@ -2487,27 +2287,21 @@ def tracked_files(repo_root: Path) -> list[str]: def discover_archives( repo_root: Path, held: frozenset[str], skip: tuple[Path, ...] = () ) -> ArchiveIndex: - """Index archives held in the repository, not ones a build unpacked. + """Index tracked or explicitly fetched archives, excluding build outputs. - `held` is what the repository tracks. It has no default: an empty set - silently indexes nothing, and a caller that forgot it would see every - archive-derived wheel member become unexplainable rather than wrong. + Require held explicitly; an omitted tracked-file list silently loses provenance. """ index = ArchiveIndex() skipped = tuple(item.resolve() for item in skip) for path in _walk(repo_root): - # By path segment, not by string prefix: `dist` prefixes `distribution` - # and a build directory named `build` prefixes `buildtools`, and an - # archive skipped that way goes on to explain nothing. + # Compare path segments: skipping dist must not skip distribution. if any(path.is_relative_to(item) for item in skipped): continue name = path.name.lower() if not name.endswith(ARCHIVE_SUFFIXES): continue relative = path.relative_to(repo_root).as_posix() - # Held here, or fetched by this build and recorded as such. An archive - # anywhere else is output -- staged into an install prefix, say -- and - # letting it explain a wheel member is the build explaining itself. + # Untracked archives need a download marker; staged copies cannot prove origin. if relative not in held and not path.with_name(path.name + ".source").is_file(): continue index.add_archive(path, relative) @@ -2519,9 +2313,7 @@ class FileIndex: def __init__(self) -> None: self.by_hash: dict[str, str] = {} - # Every path sharing a digest, because one is not evidence of origin when - # several files hold the same bytes -- a stock licence text, or a module - # that is nothing but an SPDX header. + # Keep all candidates: identical license texts or source files do not prove ownership. self.all_by_hash: dict[str, list[str]] = {} self.by_suffix: dict[str, list[str]] = {} self._digests: dict[str, str] = {} @@ -2534,8 +2326,7 @@ def digest_of(self, path: Path) -> str | None: def add_tree( self, root: Path, display_root: str, skip: tuple[Path, ...] = () ) -> None: - # Compare skip entries as paths relative to `root`: `--build-dir` may be - # relative, and `_deps/foo-srcx` string-prefixes `_deps/foo-src`. + # Resolve relative skip paths and compare segments, not string prefixes. inside = [] for item in skip: try: @@ -2559,8 +2350,7 @@ def add_paths(self, root: Path, display_root: str, paths) -> None: self._add(root, display_root, (root / item for item in paths)) def _add(self, root: Path, display_root: str, paths) -> None: - # One resolve for the tree, not one per file: `_walk` never crosses a - # symlink, so the resolved root plus the relative path is the real path. + # Resolve once per tree: _walk does not cross symlinks. resolved_root = root.resolve() for path in paths: if not path.is_file() or path.is_symlink(): @@ -2575,7 +2365,7 @@ def _add(self, root: Path, display_root: str, paths) -> None: self.by_hash.setdefault(digest, display) self.all_by_hash.setdefault(digest, []).append(display) self._by_display[display] = digest - # Two trailing segments is enough to disambiguate an __init__.py. + # Include the parent directory to distinguish common names such as __init__.py. self.by_suffix.setdefault( "/".join(relative.rsplit("/", 2)[-2:]), [] ).append(display) diff --git a/scripts/sbom/isaaccapture_sbom/document.py b/scripts/sbom/isaaccapture_sbom/document.py index 4007ef891a..656e816e49 100644 --- a/scripts/sbom/isaaccapture_sbom/document.py +++ b/scripts/sbom/isaaccapture_sbom/document.py @@ -19,14 +19,11 @@ from .wheelfile import WheelInfo NOASSERTION = "NOASSERTION" -# How a component reaches the wheel decides the relationship that says so. _LINK_HOWS = {"compiled-source", "header-include", "prebuilt-library"} -# Bytes this component also holds, which is not evidence the wheel carries its -# file. It states a candidate, so it earns no CONTAINS and no COPY_OF. +# Shared content identifies candidates, not proven CONTAINS/COPY_OF relationships. _AMBIGUOUS_HOWS = {"shared-content"} -# The package every document describes: the wheel it is about. WHEEL_PACKAGE_ID = "SPDXRef-Package-wheel" @@ -40,9 +37,9 @@ def component_package_id(key: str) -> str: def file_id(path: str, dist_info: str) -> str: - """The member's id: its path, with the versioned `.dist-info/` and `.data/` - read as `dist-info/` and `data/`, and a digest of that path, which keeps - apart paths that sanitise alike (`a_b.py`, `a-b.py`). + """Derive a stable ID from the path, normalizing versioned .dist-info/.data roots. + + Include a path digest to distinguish names that sanitize alike. """ stem = dist_info.removesuffix(".dist-info") hashed = path @@ -77,8 +74,6 @@ def _attribution_texts(component: Component, dist_info: str) -> list[str]: item.matches, name=_scored, join="; ", lead="identified as " ) if item.matches - # The SPDX reference text for an identifier the component's files - # declare, which is a substitution and not a reading of the component. else f" — the SPDX reference text for {item.identified}, an identifier " "REUSE declares for this component's files; not the component's own file" if item.origin == licensing.SPDX_REFERENCE @@ -106,8 +101,6 @@ def _component_package(component: Component, roles: set[str], dist_info: str) -> "licenseConcluded": component.license_concluded, "licenseDeclared": component.license_declared, "copyrightText": component.copyright_text, - # Only where there is one. A package identifier a reader cannot resolve - # is worse than its absence. **( { "externalRefs": [ @@ -130,7 +123,7 @@ def _component_package(component: Component, roles: set[str], dist_info: str) -> def _extracted_entries(components: list[Component]) -> list[dict]: - """One entry per distinct text, named by the id the components now share.""" + """One entry per distinct text, using the shared component LicenseRef.""" entries: dict[str, dict] = {} for component in components: carried = licensing.carried_texts(component.evidence) @@ -146,8 +139,7 @@ def _extracted_entries(components: list[Component]) -> list[dict]: f"{ref} was minted for two components with different terms; " "a LicenseRef identifies one text" ) - # Accumulated, not replaced: the entry names every component sharing - # the text and cites all of their files. + # A shared text must cite all components and evidence paths that supply it. covers = set(entries.get(ref, {}).get("_for", ())) | {component.name} cited = sorted( set(entries.get(ref, {}).get("_from", ())) @@ -179,7 +171,7 @@ def _extracted_entries(components: list[Component]) -> list[dict]: def _unanalyzed_package( package_id: str, name: str, comment: str, external_refs: list[dict] | None = None ) -> dict: - """A package this document names but did not open: everything unasserted.""" + """Describe an external package without asserting its contents.""" package = { "SPDXID": package_id, "name": name, @@ -214,11 +206,9 @@ def _by_version(specifier): def _dependency_refs( name: str, requirement: Requirement, version: str | None = None ) -> list[dict]: - """purl for the package, and the constraint as a range rather than prose. + """Represent requirements with an unversioned purl and a VERS constraint. - purl carries one exact version, which a requirement does not have. VERS (the - purl project's range notation) does, so the specifier stays machine-readable - instead of living only in a sentence. + A purl version must be exact; it cannot encode a requirement range. """ refs = [ { @@ -229,8 +219,7 @@ def _dependency_refs( ).to_string(), } ] - # VERS orders constraints by version, not by the text of the operator: the - # string sort put `<1.9` before `>=1.6` and a strict parser rejects that. + # VERS requires version ordering; sorting operators puts <1.9 before >=1.6. constraints = [ f"{item.operator}{item.version}" for item in _by_version(requirement.specifier) ] @@ -246,12 +235,9 @@ def _dependency_refs( def _source_info(project: dict, evidence_doc: dict, build: dict, wheel) -> str: - """Where this wheel came from, stating only what applies to it. + """Describe source and build facts applicable to this wheel. - One build produces several wheels, so its architecture and configuration are - facts about the compilation -- which a pure-python wheel did not have. A - placeholder rendered as a noun ("Built from NOASSERTION") is worse than an - omitted clause. + Universal wheels must not inherit the build matrix's native architecture. """ source = evidence_doc.get("project", {}) parts = [] @@ -262,11 +248,8 @@ def _source_info(project: dict, evidence_doc: dict, build: dict, wheel) -> str: described = source.get("describe") or "" parts.append( f"at commit {commit}" - # A tree with uncommitted changes did not produce what that commit - # produces, and a report nothing can reproduce has to say so. + (" with uncommitted changes" if described.endswith("-dirty") else "") ) - # `none-any` is the tag for a wheel with nothing compiled in it. if "none-any" not in wheel.path.name: parts.append(f"arch {build.get('arch')}") parts.append(f"CMAKE_BUILD_TYPE {build.get('config')}") @@ -274,20 +257,14 @@ def _source_info(project: dict, evidence_doc: dict, build: dict, wheel) -> str: def _wheel_conclusion(declared: str, inventory: Inventory) -> str: - """The declared licence, AND the terms of everything it redistributes. + """Conjoin the distribution's declaration and every redistributed component's terms. - Every component, not only the ones the build could not name: Apache-2.0 does - not cover MIT, Zlib, BSD-3-Clause, Qhull or BSL-1.0, so leaving those out - said the wheel carries no code under them. Only a component concluding the - declared licence itself adds nothing. One component with unresolved terms - leaves the whole wheel's unresolved: NOASSERTION cannot be AND-ed. + Any unresolved component makes the result NOASSERTION, which cannot be AND-joined. """ present = inventory.components_present.values() if any(item.license_concluded == NOASSERTION for item in present): return NOASSERTION - # Decomposed, so an id two components share appears once and a compound one - # does not arrive nested -- but never through an `OR`, which is one choice - # and becomes a claim on both halves if it is split. + # Flatten AND operands for deduplication while preserving OR choices. operands: set[str] = set() for component in present: operands.update(licensing.operands_of(component.license_concluded)) @@ -304,22 +281,17 @@ def build_document( excluded_files: list[str], siblings: dict[str, dict] | None = None, ) -> dict: - """Assemble the document. Callers validate it; this renders, and refuses - two members that would share an SPDXID. + """Render SPDX, rejecting file-ID collisions. - `siblings` are the other distributions of this same run, by canonical name. - A requirement naming one is not an install-time unknown: the build made it, - and its own document describes it. + The caller validates the result. siblings maps canonical distribution names + to metadata from other wheels in this run. """ files: list[dict] = [] file_ids: dict[str, str] = {} sha1_digests: list[str] = [] - # What the files themselves state, for the package-level roll-up below. stated_in_files: set[str] = set() - # Notices stated by files this distribution's own, for the package below. A - # file attributed to a component carries that component's notice, which - # belongs to that component's package and not to this one. + # Only first-party file notices contribute to the wheel package's own copyright. stated_copyright: set[str] = set() paths_by_id: dict[str, str] = {} for entry in sorted(wheel.entries, key=lambda item: item.name): @@ -345,28 +317,21 @@ def build_document( {"algorithm": "SHA256", "checksumValue": entry.sha256}, ], "licenseConcluded": NOASSERTION, - # The notice sits directly above the identifier already parsed, so - # declaring none would discard something read and in hand. "copyrightText": (attribution.copyright_text if attribution else None) or NOASSERTION, } if attribution and attribution.spdx_tag: - # A REUSE tag in the file is upstream stating that file's license. record["licenseConcluded"] = attribution.spdx_tag record["licenseInfoInFiles"] = [attribution.spdx_tag] stated_in_files.add(attribution.spdx_tag) else: - # A binary states nothing about itself, and most of this wheel is - # binaries. Listing only what was found says every file's licence - # was determined; NOASSERTION is what the rest of them found. + # Missing file declarations must remain visible in the package roll-up. stated_in_files.add(NOASSERTION) if ( attribution and attribution.copyright_text and not attribution.primary - # A packaged licence text states its component's notice, or its own - # author's -- WTFPL names the person who wrote WTFPL. Neither is a - # notice this distribution makes about itself. + # Packaged terms may name a component or license author, not this distribution. and not report.is_packaged_license(wheel.dist_info, entry.name) ): stated_copyright.add(attribution.copyright_text) @@ -387,6 +352,7 @@ def build_document( wheel_id = WHEEL_PACKAGE_ID build = evidence_doc.get("build", {}) + # The distribution declares its license; the conclusion also includes bundled terms. wheel_package = { "SPDXID": wheel_id, "name": wheel.distribution, @@ -397,18 +363,10 @@ def build_document( "downloadLocation": NOASSERTION, "filesAnalyzed": True, "homepage": project["homepage"], - # Declared is what the distribution says of itself. Concluded is what - # this document found in it, and a wheel carrying a component under - # other terms is not wholly under the declared licence -- saying so is - # the field a compliance tool reads for "what is this artifact". "licenseConcluded": _wheel_conclusion(project["license"], inventory), "licenseDeclared": project["license"], "licenseInfoFromFiles": sorted(stated_in_files) or [NOASSERTION], - # The notices its own files carry, or -- for a distribution that ships - # no file of its own -- what the project states in the metadata it is - # built from. Declaring none beside a supplier and a licence left the - # package saying less about itself than anything in it, and left a - # recipient of a metadata-only wheel unable to name its licensor at all. + # Metadata-only wheels need the project notice as a fallback. "copyrightText": "\n".join(licensing.fold_notices(stated_copyright)) or project.get("copyright", NOASSERTION), "packageVerificationCode": { @@ -445,7 +403,6 @@ def build_document( packages.append(package) package_id = package["SPDXID"] - # Linked in, copied in, or both; say whichever the evidence supports. if roles & _LINK_HOWS: relationships.append( { @@ -476,9 +433,7 @@ def build_document( } ) - # A file whose bytes several components hold names no single source, so - # it has no primary and `files_of` does not see it. Stating the candidates - # only in prose would leave it attached to nothing any tool reads. + # Ambiguous files have no primary owner; encode their candidate relationships too. for name in inventory.files_sharing(key): if name in file_ids: relationships.append( @@ -507,9 +462,8 @@ def build_document( } ) - # One node per package and version range, not per requirement line. A - # dependency recurs once per extra that wants it, and the repeats differ - # only in a marker the relationship already carries. + # Group repeated requirements by package and version range; markers belong + # to relationships, not separate package nodes. wanted: dict[tuple[str, str], list[Requirement]] = {} for requirement in inventory.requires_dist: parsed = Requirement(requirement) @@ -542,11 +496,8 @@ def build_document( ) ) else: - # Built by this same run: not an install-time unknown, so the facts - # its own metadata states are recorded here. Not a digest-bound - # reference to that wheel's document: a `py3-none-any` wheel is built - # by every matrix entry and must not differ by which one built it, - # and in a requirement cycle only one document could carry it. + # Use sibling metadata, not document hashes: universal wheels must be identical + # across matrix entries, and cyclic requirements cannot have cyclic hashes. element = package_id comment = ( f"{declared} Built by this same build; its own document " @@ -590,8 +541,6 @@ def build_document( } ) - # SOURCE_DATE_EPOCH makes the whole wheel reproducible, which is what lets a - # rebuild be compared against a published one byte for byte. stamp = stamped_now() extracted = _extracted_entries(list(inventory.components_present.values())) @@ -608,9 +557,7 @@ def build_document( f"Tool: {TOOL_NAME}-{TOOL_VERSION}", f"Organization: {project['supplier'].split(': ', 1)[-1]}", ], - # From the corpus this build identified against, never a default: a - # stated version the document was not built against is a false claim - # about every identification in it. + # Record the corpus actually used for identification, not a default version. "licenseListVersion": ".".join( str(evidence_doc["license_list_version"]).split(".")[:2] ), diff --git a/scripts/sbom/isaaccapture_sbom/elf.py b/scripts/sbom/isaaccapture_sbom/elf.py index fa948f6add..fcc6701b76 100644 --- a/scripts/sbom/isaaccapture_sbom/elf.py +++ b/scripts/sbom/isaaccapture_sbom/elf.py @@ -1,15 +1,10 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -"""SONAME, NEEDED and the GNU build-id for the shared libraries a wheel carries. +"""Read SONAME, NEEDED, and GNU build IDs with pyelftools. -Parsing is pyelftools' job, not ours -- it is the same library auditwheel uses -to repair these wheels, so the tool that wrote the ELF and the tool that reads -it agree by construction. - -Only the build path needs any of this. Consumer-side verification re-hashes -files and never parses a binary, which keeps attacker-controlled input out of a -parser on that path entirely. +Only collection parses ELF; consumer verification hashes bytes without +invoking a binary parser. """ from __future__ import annotations @@ -33,9 +28,7 @@ class DynamicInfo: soname: str | None needed: tuple[str, ...] machine: str - # Identifies the bytes the linker produced, and patchelf leaves it alone. - # So it still matches after a repair tool rewrites SONAME and RPATH, which - # is what makes it the only link back to a library's pristine origin. + # The linker's build ID survives patchelf SONAME/RPATH rewrites. build_id: str | None = None @@ -57,12 +50,7 @@ def _build_id_from(elffile: ELFFile) -> str | None: def build_id_of(data: bytes) -> str | None: - """The build-id of an in-memory image, which may be only its head. - - The note the linker writes sits with the program headers, so a bounded - prefix of a large library answers this without holding the whole file. A - prefix too short to reach the note reads as no build-id rather than raising. - """ + """Read a build ID from an ELF prefix; return None if the note lies beyond it.""" if not is_elf(data): return None try: @@ -72,12 +60,9 @@ def build_id_of(data: bytes) -> str | None: def _from_file(path: Path, read): - """`read(ELFFile)` over a file on disk, or None if it is not readable ELF. + """Apply read to an ELFFile without loading the whole binary. - pyelftools seeks, so only the headers and what `read` asks for are read -- - which matters when scanning hundreds of a machine's libraries, some of - them hundreds of megabytes. Scanning also meets files that only look like - ELF, and a truncated one raises `struct.error` from the parser. + Unreadable or truncated ELF returns None, including parser struct.error failures. """ try: with path.open("rb") as handle: @@ -105,8 +90,7 @@ def read_dynamic(data: bytes) -> DynamicInfo: soname: str | None = None needed: list[str] = [] - # The dynamic segment, not the section: a stripped library keeps its program - # headers and loses its section headers, and the loader reads the segment. + # Read the dynamic segment: stripped libraries can lack section headers. for segment in elffile.iter_segments(type="PT_DYNAMIC"): for tag in segment.iter_tags(): kind = tag.entry.d_tag diff --git a/scripts/sbom/isaaccapture_sbom/evidence.py b/scripts/sbom/isaaccapture_sbom/evidence.py index 89d1510e14..30f7b59e50 100644 --- a/scripts/sbom/isaaccapture_sbom/evidence.py +++ b/scripts/sbom/isaaccapture_sbom/evidence.py @@ -36,8 +36,7 @@ EMPTY_SHA256, ) -# Cache entries that pin what a wheel contains. The rest of the cache is paths on -# the builder. +# Record reproducibility settings without unrelated builder-local cache paths. _CACHE_KEYS = re.compile( r"^(CMAKE_BUILD_TYPE|CMAKE_CXX_COMPILER|CMAKE_C_COMPILER|CMAKE_SYSTEM_PROCESSOR" r"|CMAKE_CXX_COMPILER_VERSION|ISAAC_TELEOP_PYTHON_VERSION|BUILD_[A-Z0-9_]+" @@ -87,11 +86,7 @@ def _cmake_cache(build_dir: Path) -> dict[str, str]: def _fetched_from(archive: Path) -> str | None: - """The URL a fetch script recorded beside an archive it downloaded. - - Most archives here are downloaded at configure time rather than checked in, - so the origin is only knowable if whatever fetched it wrote the URL down. - """ + """Read the download URL preserved beside an archive as a .source marker.""" marker = archive.with_name(archive.name + ".source") if not marker.is_file(): return None @@ -100,19 +95,12 @@ def _fetched_from(archive: Path) -> str | None: def _supplier_of(url: str | None) -> str: - """Who supplied the archive, where the URL says so. - - A hostname is not a supplier: `api.ngc.nvidia.com` is a CDN endpoint, and - SPDX `supplier` is who a recipient asks for source or terms. Only a host - whose registrable domain names the supplier answers that, and the download - location carries the URL for everything else. - """ + """Infer a supplier from the URL's penultimate host label when available.""" host = urlparse(url).hostname if url else None if not host: return "NOASSERTION" labels = host.split(".") - # Second-level label of a two-label public suffix (`nvidia` of - # `api.ngc.nvidia.com`); anything shorter is not a name. + # For api.ngc.nvidia.com this yields nvidia; this is not a public-suffix parser. name = labels[-2] if len(labels) >= 2 else "" return f"Organization: {name}" if len(name) > 2 else "NOASSERTION" @@ -120,7 +108,7 @@ def _supplier_of(url: str | None) -> str: def _archive_components( archives: ArchiveIndex, repo_root: Path ) -> dict[str, Component]: - """A tarball in the repo is a component: its members ship as they are.""" + """Describe each indexed archive with its root license evidence.""" components: dict[str, Component] = {} for display, record in archives.archives.items(): name = Path(display).name @@ -138,8 +126,7 @@ def _archive_components( ) for member, data in record.get("license_members", []) ] - # `display`, not `name`: the name is the archive's basename, so two - # archives in different directories would share a LicenseRef. + # Use the full display path to distinguish archives with the same basename. concluded, declared = licensing.expression(evidence, display) source_url = _fetched_from(repo_root / display) @@ -176,7 +163,7 @@ def _archive_version(archives: ArchiveIndex, display: str) -> str | None: def discover(repo_root: Path, build_dir: Path) -> Discovery: - """Index a configured build tree. Nothing here is declared anywhere.""" + """Index a configured build tree.""" cache = _cmake_cache(build_dir) deps_dir = build_dir / "_deps" if not deps_dir.is_dir(): @@ -184,9 +171,7 @@ def discover(repo_root: Path, build_dir: Path) -> Discovery: f"{deps_dir} not found; configure the build before collecting evidence" ) - # Guessing a configuration would point the build graph and the staged-tree - # probe at a directory that need not exist, and every member they would have - # explained then reads as having reached the wheel by an unknown route. + # Require the configuration that produced the graph and staged package tree. config = cache.get("CMAKE_BUILD_TYPE") if not config: raise EvidenceError( @@ -194,11 +179,7 @@ def discover(repo_root: Path, build_dir: Path) -> Discovery: "graph and the staged tree are both read per configuration" ) graph = BuildGraph(build_dir, config) - # Everything that is a build tree rather than source, skipped the same way - # everywhere it matters. The active one is given; any other is found by the - # CMakeCache.txt that makes it one. A second build directory holds staged - # copies of the repo, and walking those as source leaves a file's origin - # ambiguous rather than wrong, which surfaces as an unexplained member. + # Exclude other build trees too; their staged copies create false source matches. not_source = ( build_dir, repo_root / "build-wheel", @@ -220,21 +201,13 @@ def discover(repo_root: Path, build_dir: Path) -> Discovery: if root.is_dir(): source_files.add_tree(root, root.name) - # Tracked files only: an install prefix carries no marker distinguishing it - # from source, so a directory walk cannot, and a wheel member matching this - # build's own output would be reported as a copy of repository source. + # Only tracked files are repository sources; install prefixes have no identifying marker. repo_files = FileIndex() repo_files.add_paths(repo_root, ".", tracked) - # _deps/*-src is the component-source domain, already indexed above; a wheel - # member matching one is a copy of upstream, not something this build made. - # Everything else under _deps is this build's own output -- FetchContent - # build trees, and trees a dependency stages for packaging -- so it belongs - # in build_files like any other generated file. The staged tree stays out: - # indexing it would let the wheel explain itself. - # Named after the directory that was read: `--build-dir` is free-form, and a - # citation reading `build/...` resolves to the wrong tree, or to nothing, for - # anyone whose build directory is not called that. + # Index generated output, including _deps/*-build, separately from fetched sources. + # Exclude staging so the wheel cannot explain itself. Preserve the actual build + # directory name in evidence paths. build_files = FileIndex() build_files.add_tree( build_dir, @@ -242,22 +215,16 @@ def discover(repo_root: Path, build_dir: Path) -> Discovery: skip=(*sorted(deps_dir.glob("*-src")), build_dir / "python_package"), ) - # A dependency may stage a packaged copy of its own tree under _deps//. - # Files the build transformed on the way in no longer hash to the component, - # so infer the directory's owner from the siblings that still do; one clear - # owner or nothing, since a guess here would invent a license obligation. + # Infer staged-tree ownership from unchanged siblings after transformations. + # Require a unique owner to avoid inventing a license obligation. staged_component_dirs: dict[str, str] = {} for root in sorted(deps_dir.iterdir()): if not root.is_dir() or root.name.endswith("-src"): continue display = f"{build_dir.name}/{root.relative_to(build_dir).as_posix()}" owners = set() - # From the index rather than a second walk: `build_files` already hashed - # every file under the build tree. for _, digest in build_files.under(display): - # Zero bytes match every other empty file, so a CMake scaffolding - # directory full of stamps would adopt whichever component happened - # to ship an empty file. They are evidence of nothing here too. + # Empty files cannot identify an owner. if digest == EMPTY_SHA256: continue match = source_files.by_hash.get(digest) @@ -270,16 +237,11 @@ def discover(repo_root: Path, build_dir: Path) -> Discovery: for root in sorted(deps_dir.glob("*-src")): if root.is_dir(): ownership.register_root(root, root.name.removesuffix("-src")) - # A dependency generates sources into its own build tree -- glfw's Wayland - # protocol headers, Catch2's generated config. Those sit under the build - # directory and belong to no *-src checkout, so without this they count as - # this project's own sources and the target reads as first-party. + # Generated dependency headers belong to their component, not the root project. for root in sorted(deps_dir.glob("*-build")): if root.is_dir(): ownership.register_root(root, root.name.removesuffix("-build")) - # What vcpkg built, where VCPKG_INSTALLED_DIR (by default the build tree's - # vcpkg_installed) records which port installed each file. ports, port_files = discover_vcpkg( Path(cache.get("VCPKG_INSTALLED_DIR") or build_dir / "vcpkg_installed"), cache.get("VCPKG_TARGET_TRIPLET"), @@ -289,8 +251,7 @@ def discover(repo_root: Path, build_dir: Path) -> Discovery: for path in paths: ownership.register_file(path, key, by_directory=False) - # Third-party material checked in here rather than fetched. Tracked files as - # well as build inputs: Python or data reaching the wheel is compiled by nothing. + # Include tracked data and Python as well as native build inputs. vendored, owned_paths = discover_vendored( repo_root, graph.repo_inputs(repo_root, build_dir), @@ -321,11 +282,9 @@ def discover(repo_root: Path, build_dir: Path) -> Discovery: def document( discovery: Discovery, wheel_name: str, inventory, components: dict[str, Component] ) -> dict: - """The build-evidence sidecar: resolved facts, not the raw indexes. + """Render build evidence using this wheel's resolved components. - `components` is what this wheel resolved against, which is the discovered set - plus the system libraries its own members turned out to need. One `Discovery` - serves every wheel in a run, so it cannot hold those. + Keep host libraries per wheel; Discovery is shared across the run. """ repo_root = discovery.repo_root arch = { @@ -378,8 +337,6 @@ def document( }, "build": { "build_dir": str(discovery.build_dir), - # Named once: a member's origin reads "staged" rather than repeating - # this prefix and the member's own path for every file staged here. "staged_root": ( str(discovery.staged_root) if discovery.staged_root else None ), @@ -403,10 +360,7 @@ def document( "license_concluded": component.license_concluded, "license_declared": component.license_declared, "evidence": [item.as_json() for item in component.evidence], - # The digest of the terms verbatim, as the document carries - # them. A component can state its licence across several files, - # and the join has no digest of its own for the verifier to - # compare against unless the build states one. + # Bind the combined terms as well as individual files so LicenseRef text is verifiable. "verbatim_terms_sha256": hashlib.sha256( licensing.verbatim_terms(component.evidence).encode("utf-8") ).hexdigest(), diff --git a/scripts/sbom/isaaccapture_sbom/inventory.py b/scripts/sbom/isaaccapture_sbom/inventory.py index 928dba0f47..3807399ca1 100644 --- a/scripts/sbom/isaaccapture_sbom/inventory.py +++ b/scripts/sbom/isaaccapture_sbom/inventory.py @@ -1,13 +1,7 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -"""Account for every file in a wheel by where it came from. - -Each member is traced to a link step, a fetched source tree, an archive in the -repo, the working tree, or the build's own generated output. A member that none -of those explain is an error: it means something reached the wheel by a route -this collector cannot see, and it stops publication. -""" +"""Attribute wheel members to build inputs; unexplained members block publication.""" from __future__ import annotations @@ -37,7 +31,7 @@ if TYPE_CHECKING: # pragma: no cover from .evidence import Discovery -# Text-ish members worth reading a REUSE tag out of. +# File types whose contents may carry REUSE tags. _TAGGABLE = { ".py", ".pyi", @@ -63,17 +57,11 @@ class Attribution: primary: str | None = None spdx_tag: str | None = None copyright_text: str | None = None - # Whether the member is those bytes, as opposed to something made from them. - # Kept apart from `origin`, which says where it came from: inferring one from - # the other made the same fact answerable two ways depending on which index - # happened to hold the bytes. + # Byte identity is independent of origin; do not infer it from the attribution route. exact: bool = False - # Where in the component these bytes came from, where an index named it: - # the detail string's fact in a form the evidence JSON can carry. + # Path within the component, when known. source_path: str | None = None - # Each component's paths these bytes were made from -- the file copied, or - # what compiling it read -- named as `source_path` names them, with their - # `discovery.REACH`. + # Component -> source path -> discovery.REACH method. read: dict[str, dict[str, str]] = field(default_factory=dict) def as_json(self) -> dict: @@ -110,7 +98,7 @@ def files_sharing(self, key: str) -> list[str]: return self._by_component()[1].get(key, []) def _by_component(self) -> tuple[dict[str, list[str]], dict[str, list[str]]]: - """One pass over the members, not one per component per caller.""" + """Cache the component index to avoid rescanning members for each lookup.""" if self._indexed is None: owned: dict[str, list[str]] = {} shared: dict[str, list[str]] = {} @@ -132,16 +120,13 @@ class Resolver: def __init__(self, discovery: Discovery, system_resolver=None) -> None: self._system_resolver = system_resolver or resolve_system_library self.system_libraries: dict[str, dict] = {} - # Archives and objects this wheel's members link that no target - # produced, by path, with what explains each -- or `explained: False` - # for the build to refuse. + # External static inputs by path; unexplained entries fail the build gate. self.static_inputs: dict[str, dict] = {} self._input_keys: dict[str, str] = {} self.repo_root = discovery.repo_root self.build_dir = discovery.build_dir self.graph = discovery.graph - # A copy: `_from_build_host` adds the system libraries this wheel turned - # out to need, and one `Discovery` is shared by every wheel in a run. + # Copy before adding host libraries: Discovery is shared across wheels. self.components = dict(discovery.components) self.archives = discovery.archives self.repo_files = discovery.repo_files @@ -167,8 +152,6 @@ def __init__(self, discovery: Discovery, system_resolver=None) -> None: ) def _from_artifact(self, artifact, path: str, detail: str) -> Attribution: - # What a fetched tree, a vcpkg port or vendored material holds is - # credited there; what is left is the build machine's, or the gate's. contributions = self.graph.contributions(artifact, self.ownership) linked = self.graph.static_inputs(artifact) for item in sorted(linked, key=lambda item: (item.key, item.named)): @@ -200,14 +183,10 @@ def _from_artifact(self, artifact, path: str, detail: str) -> Attribution: def _displayed( self, reached: dict[str, dict[Path, str]] ) -> dict[str, dict[str, str]]: - """Reached paths as a component's own files are named: `/`. - - Only a component with a tree of its own has a name for them, and only - such a component states terms for part of itself. A file the build - copied or generated into another tree registered for the component -- - its `-build` tree -- stands where it sits relative to that tree, and is - also any file of the source tree holding the same bytes. An include - directory holding a whole tree stands for the component's root. + """Render reached paths as /. + + Build-tree files retain their position and any byte-identical source paths. + An include directory enclosing the tree represents the component root. """ shown: dict[str, dict[str, str]] = {} for key, paths in reached.items(): @@ -248,7 +227,6 @@ def _names_in(self, name: str, built: bool, tree: Path, path: Path) -> list[str] return [*same, positional] def _static_input(self, item: LinkInput, targets: set[str]) -> str | None: - """The component a linked archive or object belongs to, if any.""" record = self.static_inputs.get(item.key) if record is None: record = self.static_inputs[item.key] = self._input_record(item) @@ -257,8 +235,7 @@ def _static_input(self, item: LinkInput, targets: set[str]) -> str | None: return record["component"] def _input_record(self, item: LinkInput) -> dict: - """What explains one input the ownership index does not: the repository, - the host, or nothing -- which the build refuses.""" + """Resolve an unowned static input against the repository and build host.""" record: dict = { "kind": item.kind, "named_as": [], @@ -286,8 +263,7 @@ def refused(outcome: str) -> dict: "records which component installed it" ) if self.repo_files.digest_of(path): - # REUSE names a holder for every tracked file, and the vendored - # scan registers one that is not this project. + # The vendored scan registers third-party holders; remaining tracked inputs are ours. record["outcome"] = "tracked in this repository, held by this project" return record @@ -315,10 +291,7 @@ def refused(outcome: str) -> dict: return record def _input_key(self, path: Path) -> str: - """`system:`, unless another linked file already holds that name. - - A shared library's key is its SONAME, which no archive or object shares. - """ + """Mint a system key, distinguishing static files with the same basename.""" key = f"system:{path.name}" if self._input_keys.get(key, str(path)) != str(path): key += "@" + hashlib.sha256(str(path).encode()).hexdigest()[:8] @@ -339,17 +312,11 @@ def resolve( patched = False if probe is not None and probe.is_file(): candidate_digest = sha256_file(probe) - # The staged file is what the build put there; the member is what the - # wheel ships. Tracing from the staged bytes finds the origin, but - # the two can differ -- a repair tool rewrites members after staging - # -- and saying the member is a copy of something it does not hash to - # would describe a file the wheel does not contain. + # Staged bytes reveal origin; repaired wheel bytes may differ and must be + # reported as derived rather than copied. patched = candidate_digest != digest relative = probe.relative_to(self.build_dir).as_posix() - # The staged path is the staging root plus the member's own path - # unless the build renamed the file on the way in. Printing it - # whenever it is derivable repeats the member path once per member - # and says nothing; where it is not, it is the interesting part. + # Show the staged path only when it differs from the wheel member's path. source = ( "staged" if relative.endswith(f"/{wheel_path}") @@ -380,11 +347,7 @@ def _trace( soname: str | None, build_id: str | None, ) -> Attribution | None: - """Find what explains these bytes, in order of how much it proves.""" - # Zero bytes are identical everywhere, so a content match proves nothing: - # it would hand an empty marker file to whichever component happened to be - # indexed first, and invent that component's license obligation with it. - # The path is the only evidence such a file carries. + # Empty content matches every empty file; only a path can establish its origin. if candidate_digest == EMPTY_SHA256: return self._resolve_by_path(wheel_path, source, build_id, candidate_digest) @@ -394,10 +357,8 @@ def _trace( artifacts[0], wheel_path, f"{source}; exact build output" ) - # Identical bytes are not evidence of origin when several components - # hold them: a stock Apache-2.0 text is the same file in every project - # that ships one. Say so, and let the path decide, rather than naming - # whichever happened to be indexed first. + # Shared bytes (for example a standard license text) do not prove ownership. + # Use the path to disambiguate or retain every candidate. shared = self._shared_content(candidate_digest) if shared is not None: by_path = self._resolve_by_path( @@ -438,19 +399,13 @@ def _trace( build_file = self.build_files.by_hash.get(candidate_digest) if build_file: - # An empty file hashes the same as every other empty file, so the - # first indexed path wins and says nothing. Where the build tree - # holds the same bytes under the wheel's own path, that is the - # better answer. + # Prefer the wheel's path when several build files contain identical bytes. if Path(build_file).name != Path(wheel_path).name: by_path = self.build_files.path_suffix_match(wheel_path) if by_path: build_file = by_path owner = self._staged_component_of(build_file) - # Say which of the two this is. The bytes match the build tree, so - # the member is that file; what the build made it from is not - # recorded anywhere this can read, and claiming a derivation without - # naming its input is the claim the document cannot support. + # A build-tree match proves a copy, but does not identify the source it was made from. detail = f"{source}; byte-identical to {build_file}" if owner: detail += ( @@ -459,18 +414,14 @@ def _trace( ) return Attribution( path=wheel_path, - # Transforming a component's own file leaves a derivative work, - # which carries that component's terms. Only output that owes - # nothing to a third party is merely "generated". + # Generated output owned by a component still carries that component's terms. origin="derived" if owner else "generated", detail=detail, components={owner: {"derived-file"}} if owner else {}, primary=owner, ) - # Content no longer matches, so the bytes were patched on the way in. - # The build-id is what the linker wrote and patchelf leaves alone, which - # makes it evidence where a matching file name is only a coincidence. + # Build IDs survive repair and provide stronger evidence than a matching name. if build_id: by_build_id = self._resolve_by_build_id(wheel_path, source, build_id) if by_build_id is not None: @@ -485,9 +436,7 @@ def _trace( if from_host is not None: return from_host - # Nothing this build produced or fetched explains these bytes. A shared - # library reaching that point came from the machine -- whatever the - # repair tool chose to call its directory. + # Consult the host last, regardless of the directory auditwheel used. if soname: return self._from_build_host(wheel_path, soname) return None @@ -495,13 +444,7 @@ def _trace( def _from_repo_file( self, wheel_path: str, repo_file: str, lead: str, same: bool ) -> Attribution: - """A file this repository holds -- and, where it is not ours, whose. - - Checked-in third-party material declares its own copyright holder, - which is how the vendored scan found it. Reporting the member as plain - repository source would drop that holder, and the component would be - reported as contributing nothing to a wheel that ships its file. - """ + """Attribute repository files while retaining any third-party ownership.""" owner = self.ownership.owner_of(self.repo_root / repo_file.removeprefix("./")) return Attribution( path=wheel_path, @@ -522,12 +465,7 @@ def _staged_component_of(self, build_file: str) -> str | None: return None def _from_build_host(self, wheel_path: str, soname: str) -> Attribution | None: - """Claim host origin only if the host actually has this library. - - Resolving the SONAME is the evidence for the claim. Without it there is - nothing behind "the machine supplied this", and the member is better - reported as unexplained than attributed to a library no one found. - """ + """Require a resolved host library before claiming host origin.""" origin = self._system_resolver(soname) if not origin.get("resolved_path"): return None @@ -549,22 +487,16 @@ def _from_build_host(self, wheel_path: str, soname: str) -> Attribution | None: ) def _shared_content(self, digest: str): - """How to describe content that more than one component also holds. + """Return an attribution builder for ambiguous content, otherwise None. - Returns None when the digest names exactly one component, which is the - case worth stating as a fact. Otherwise it returns a builder that names - every candidate and claims none of them, keeping each component's - licence obligation without asserting a copy that did not happen. + Retain every candidate's terms without claiming one as the source. """ owners: dict[str, str] = {} for display in self.source_files.all_by_hash.get(digest, []): owners[component_of(display)] = display for member in self.archives.all_by_hash.get(digest, []): owners[member.container] = f"{member.path} in {member.container}" - # This repository holding the same bytes counts as a candidate too: a - # module that is nothing but an SPDX header is identical in every project - # that uses the same header, and calling ours a copy of a dependency's - # would ship a first-party file as redistributed third-party content. + # Include first-party matches so identical headers cannot imply third-party origin. ours = self.repo_files.all_by_hash.get(digest, []) if len(owners) + bool(ours) < 2: return None @@ -587,10 +519,7 @@ def _resolve_by_build_id( self, wheel_path: str, source: str, build_id: str ) -> Attribution | None: """The pristine bytes this member was patched from, named by build-id.""" - # The same library can sit in two archives under two sets of terms, and - # SDKs in this build do share build-ids. Every neighbouring index refuses - # to name one of several candidates; this one must too, or a patched - # member takes whichever archive happened to be walked first. + # SDK archives can share build IDs under different terms; retain all candidates. members = self.archives.all_by_build_id.get(build_id, []) containers = {item.container for item in members} member = members[0] if len(containers) == 1 else None @@ -645,15 +574,10 @@ def _resolve_by_path( build_id: str | None = None, digest: str | None = None, ) -> Attribution | None: - """Post-processing changes bytes; the path still says where they came from. + """Use path evidence when content hashes cannot identify an origin. - patchelf, auditwheel's RPATH rewrite and the MJCF mesh stripping all leave - a file that no longer hashes to its origin, so a unique path match is the - remaining evidence -- recorded as derived, never as an exact copy. - - A name match is only ever a guess, so a build-id that disagrees overrules - it: two libraries called libcloudxr.so from different SDKs are not each - other, and claiming they are would hand one the other's licence. + Repaired binaries and stripped MJCF assets can differ from their sources. + Conflicting known build IDs override a name match; equal bytes remain copies. """ name = Path(wheel_path).name @@ -686,12 +610,8 @@ def _resolve_by_path( artifacts[0], wheel_path, f"{source}; modified build output" ) - # This repository first. A member matching a file this project tracks is - # this project's copy; a dependency whose tree happens to end in the same - # two path segments is a coincidence, as when an SDK ships a BSL-1.0 - # text under the same relative path as our pool's. - # A third-party file checked in here still carries its component, because - # the repository branch looks its holder up. + # Prefer tracked repository paths over coincident dependency suffixes. + # Third-party ownership of tracked files is still preserved. repo_file = self.repo_files.path_suffix_match(wheel_path) if repo_file: same = self.repo_files.digest_of_display(repo_file) == digest @@ -702,10 +622,7 @@ def _resolve_by_path( if component_file: key = component_of(component_file) origin_digest = self.source_files.digest_of_display(component_file) - # Reaching this branch means the content matched more than one place, - # not that anything transformed the file. Where the input's bytes are - # these bytes it is a copy, and quoting an origin digest equal to the - # member's own would say nothing. + # Path disambiguation does not imply modification; compare the bytes. same = origin_digest == digest return Attribution( path=wheel_path, @@ -747,7 +664,6 @@ def build( resolver: Resolver, staged_root: Path | None, ) -> Inventory: - """Attribute every member, then fold the result into component presence.""" attributions: dict[str, Attribution] = {} unattributed: list[str] = [] provided_sonames = {entry.soname for entry in wheel.entries if entry.soname} @@ -803,15 +719,10 @@ def _build_ids_disagree(left: str | None, right: str | None) -> bool: def _resolve_shared_by_sibling(attributions: dict[str, Attribution]) -> None: - """Let a staged directory settle what identical bytes cannot. - - `libcloudxr.so` differs between two SDKs and resolves by build-id, but the - libraries beside it are byte-identical in both, and MuJoCo's licence text is - the same file every Apache-2.0 project ships. In each case the directory the - build staged the member into is already explained by exactly one of the - candidates, which is a record of where it came from -- so saying the build - kept none would be untrue, and naming every candidate overstates what the - others contribute. + """Resolve identical content using a staged sibling with one known owner. + + For example, SDKs may share libraries but have different libcloudxr.so build + IDs. The uniquely attributed sibling identifies which SDK was staged. """ settled: dict[str, set[str]] = {} for name, item in attributions.items(): @@ -835,11 +746,8 @@ def _resolve_shared_by_sibling(attributions: dict[str, Attribution]) -> None: item.primary = key -# What a packaging backend writes, and SBOM documents other tools leave here. -# `licenses/` is deliberately absent: the project's own texts are copies of files -# this repository tracks, so they trace like any other copy, and the texts this -# collector adds are attributed as it adds them. Calling the directory metadata -# asserted an origin for files that have a real one. +# Only known backend/tool output is metadata. License files must still trace +# to their sources or to the collector that added them. _DIST_INFO_WRITTEN = re.compile( r"^(METADATA|WHEEL|RECORD|INSTALLER|REQUESTED|entry_points\.txt|top_level\.txt" r"|direct_url\.json|zip-safe|namespace_packages\.txt)$|^sboms/" @@ -854,10 +762,7 @@ def _attribute( ) -> Attribution | None: if entry.name.startswith(f"{wheel.dist_info}/"): relative = entry.name[len(wheel.dist_info) + 1 :] - # Only what a packaging backend and this collector actually write there. - # Treating the whole directory as metadata asserted an origin for - # anything placed in it, which is the one route around the gate that - # every other member has to pass. + # Do not exempt arbitrary .dist-info files from provenance checks. if _DIST_INFO_WRITTEN.match(relative): return Attribution( path=entry.name, @@ -873,7 +778,6 @@ def _attribute( def _head_facts(entry: Entry) -> tuple[str | None, str | None]: - """The tag and notice a file states about itself: upstream, about that file.""" if Path(entry.name).suffix.lower() not in _TAGGABLE: return None, None head = entry.head.decode("utf-8", "replace") diff --git a/scripts/sbom/isaaccapture_sbom/licensing.py b/scripts/sbom/isaaccapture_sbom/licensing.py index e2de110432..a4f5308f39 100644 --- a/scripts/sbom/isaaccapture_sbom/licensing.py +++ b/scripts/sbom/isaaccapture_sbom/licensing.py @@ -1,11 +1,9 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -"""Find the license texts a component ships, and name what they are. +"""Identify component license texts against the SPDX corpus. -Nothing here declares a license. Texts are discovered in the component's own -files and matched against the SPDX reference corpus; a text that matches nothing -is still packaged and reported as unidentified, never guessed at. +Preserve unmatched terms verbatim and report them as unidentified. """ from __future__ import annotations @@ -31,10 +29,8 @@ from .discovery import Reach -# Root-level names that carry a grant, and the ones that carry notices instead. -# Both are packaged; only the first kind is matched against the corpus. -# The word has to stand alone: BSD-LICENSE and LICENSE.md are grants, while -# licenses.md and accessingLicenses.md are indexes that merely mention them. +# Match standalone grant names: BSD-LICENSE and LICENSE.md, not license indexes. +# Notices are packaged separately and do not determine the license expression. _GRANT_NAMES = re.compile( r"(^|[-_.])(LICEN[CS]E|COPYING|UNLICENSE|EULA)([-_.]|$)", re.IGNORECASE ) @@ -47,16 +43,11 @@ _NGRAM = 6 # A canonical text has to be almost entirely present before it is named. _CONTAINMENT_THRESHOLD = 0.90 -# ...and has to account for a real share of the file it was found in. Large -# agreements (the CUDA EULA, for one) reproduce whole OSS licenses in an -# appendix; those reach full containment at a few percent coverage, while a -# genuine match sits well above it -- a Debian copyright file naming several -# licenses around the one matched is the low end, at a quarter. +# Coverage rejects an OSS license quoted in a larger agreement's appendix +# (for example CUDA's EULA) as the agreement's own terms. _COVERAGE_THRESHOLD = 0.20 -# A grant whose matches account for less of it than this, its copyright notices -# aside, is carried as well as named: the rest may be terms no id names -- fmt's -# MIT carries an exception paragraph. A holder list is not terms, so it does not -# count against the match. Not a gate on naming. +# Carry partially explained grants too: unmatched prose may add terms, as +# in fmt's MIT exception. Exclude copyright notices from this coverage measure. _EXPLAINED_THRESHOLD = 0.80 # Two licenses are one family when either text is nearly inside the other. _FAMILY_THRESHOLD = 0.90 @@ -64,9 +55,7 @@ _SPDX_TAG = re.compile(r"SPDX-License-Identifier:\s*(?P[^\n\r*/#]+)") _FILE_COPYRIGHT = re.compile(r"SPDX-FileCopyrightText:\s*(?P[^\n\r]+)") # REUSE-IgnoreEnd -# Heading, then everything up to the next heading or a blank-line run. -# The heading has to BE about licensing, not merely start with the word -- -# "# License List Data" is a project title, not a grant. +# Require a licensing heading; a title such as "License List Data" is not a grant. _README_LICENSE_SECTION = re.compile( r"^#{1,6}\s*(licen[cs]e|licen[cs]es|licensing|licen[cs]e information|copyright|legal)" r"\s*$(?P.*?)(?=^#{1,6}\s|\Z)", @@ -94,9 +83,8 @@ class LicenseEvidence: identified: str | None = None matches: tuple[tuple[str, float, float], ...] = () spdx_tag: str | None = None - # For "nested" terms: what they cover, as a path in the component -- their - # directory, or the one file a REUSE `.license` sidecar annotates -- and, - # where a wheel keeps them, the strongest evidence that it reached a member. + # Nested terms cover a directory or one sidecar-annotated file. reached_by + # records the strongest evidence that this scope contributed to the wheel. scope: str | None = None reached_by: Reach | None = None @@ -124,15 +112,12 @@ def as_json(self) -> dict: return found -# What counts as a text that discharges the packaging obligation. A REUSE pool -# names no single expression for the component and is still a real text, so it -# counts. The build gate, the report and the verifier all ask this question; -# they ask it here. +# A REUSE pool satisfies text packaging even when no component-wide license +# can be concluded. Share this predicate across build, report, and verification. OBLIGATION_KINDS = frozenset({"grant", "pool"}) def satisfies_obligation(kind: str) -> bool: - """For callers holding the JSON form, where there is no dataclass to filter.""" return kind in OBLIGATION_KINDS @@ -141,17 +126,14 @@ def obligation_texts(evidence) -> list[LicenseEvidence]: def grant_texts(evidence) -> list[LicenseEvidence]: - """The texts a component's own licence is read from.""" return [item for item in evidence if item.kind == "grant"] def pool_texts(evidence) -> list[LicenseEvidence]: - """A REUSE pool's texts: what individual files are under, not the component.""" return [item for item in evidence if item.kind == "pool"] def pool_names(paths) -> str: - """A pool's texts by the licence each is named for, as `MIT, Zlib`.""" return ", ".join(sorted(Path(path).stem for path in paths)) @@ -166,12 +148,9 @@ def carried_and_named(evidence) -> list[LicenseEvidence]: def verbatim_terms(evidence) -> str: - """The body a LicenseRef carries for a component, from `carried_texts`. + """Join carried grants, or all grants if none need carrying. - One component can state its terms across several files, so this is a join - and not a file. The build records its digest and the verifier compares - against that, which only works while both sides join the same way -- so they - both call this. A component with nothing carried states every grant. + Build and verification must use identical joins when binding these terms by hash. """ texts = carried_texts(evidence) or grant_texts(evidence) return "\n\n".join(item.text for item in texts) @@ -181,12 +160,10 @@ def verbatim_terms(evidence) -> str: def _normalize(text: str) -> list[str]: - """Reduce a license to comparable boilerplate. + """Normalize years and clause numbers for matching. - Years and clause numbers are the part that differs between two copies of one - license, so digits come out. Copyright lines stay: BSD clause 3 and MIT's - notice-retention clause both contain the word, and upstream wrapping differs - from SPDX's, so dropping those lines loses real text asymmetrically. + Retain copyright lines: BSD and MIT clauses contain that word, and deleting + whole lines would remove terms differently depending on upstream wrapping. """ return [ word for word in _WORD.sub(" ", _DIGITS.sub(" ", text.lower())).split() if word @@ -204,12 +181,7 @@ def _ngrams(words: list[str]) -> set[str]: @dataclass(frozen=True) class Reference: - """One SPDX licence, normalized for comparison and kept verbatim. - - The text is retained because it is the only authoritative copy of a licence - this build has: a component whose terms are declared as an identifier has - to be packaged with the text that identifier names. - """ + """An SPDX reference with n-grams for matching and verbatim text for packaging.""" name: str grams: frozenset[str] @@ -226,10 +198,8 @@ class CorpusError(Exception): _CORPUS: tuple[str, dict[str, Reference]] | None = None _CORPUS_META: dict[str, object] = {} -# Every licence and exception id the same list names, deprecated ones included. _LISTED: "_Listed | None" = None -# Parsing and n-gramming the whole SPDX list costs about half a second. Keyed by -# directory and only ever read within one process, so a re-load is free. +# Cache parsing and n-gram construction by corpus directory within this process. _PARSED: dict[str, tuple[str, dict[str, Reference], "_Listed"]] = {} @@ -249,13 +219,10 @@ def corpus_provenance() -> dict[str, object]: def load_corpus(json_dir: Path) -> str: - """Load SPDX reference texts from the checkout deps/third_party fetched. + """Load the pinned SPDX corpus from its json/ directory. - `json_dir` is the `json/` directory of spdx/license-list-data: the licence - reference texts, every licence and exception id the list names, and the - exceptions' texts. Deprecated licence ids are - skipped: naming a component with one would be reporting a license SPDX has - withdrawn. + Retain all identifiers and exception texts for declaration checks; exclude + deprecated license IDs from matching candidates. """ global _CORPUS, _LISTED details = json_dir / "details" @@ -341,7 +308,7 @@ def _checkout_commit(source_dir: Path) -> str: def corpus() -> tuple[str, dict[str, Reference]]: - """SPDX reference texts as (list version, {id: (name, n-grams)}).""" + """Return (license-list version, references indexed by identifier).""" if _CORPUS is None: raise CorpusError( "SPDX license list data has not been loaded; it is fetched by the build " @@ -351,16 +318,10 @@ def corpus() -> tuple[str, dict[str, Reference]]: def identify(text: str) -> list[tuple[str, float, float]]: - """Name the licenses a text contains, as (id, containment, coverage). - - Containment says how much of a canonical license is present; coverage says - how much of this file that accounts for. Both must clear their threshold, so - a license quoted in an appendix is not mistaken for the file's own terms. A - file really can carry two licenses -- dual-licensed sources do -- so this - returns every match that survives the family resolution below. + """Return (id, canonical-text containment, input-text coverage) matches. - Compare every reference; do not prune by length. Pruning ties correctness to - the accept conditions never growing, and fails by not identifying a license. + Both thresholds must pass. Keep distinct matches after family resolution; + do not prune by length, which can miss matches as acceptance thresholds change. """ candidate = _ngrams(_normalize(text)) if not candidate: @@ -393,15 +354,10 @@ def _dominated( scored: dict[str, tuple[float, float, float]], entries: dict[str, Reference], ) -> set[str]: - """Within a family of near-identical licenses, keep only the best fit. - - Whole families differ by one clause: BSD-2 inside BSD-3, Apache-2.0 inside - ECL-2.0, CC-BY-4.0 inside CC-BY-NC-4.0. Every one of those clears the - containment threshold against a file that is really the other, so the - discriminator has to be which reference explains the whole file *and nothing - more* -- the highest Jaccard similarity. Preferring the longer text instead - would report Apache-2.0 code as ECL-2.0, and CC-BY-4.0 as its - non-commercial variant. + """Keep the highest Jaccard match within near-identical license families. + + Length alone would favor ECL-2.0 over Apache-2.0, or CC-BY-NC over CC-BY, + even when the input contains no additional terms. """ dropped: set[str] = set() for left in scored: @@ -422,11 +378,9 @@ def _family(entries: dict[str, Reference], left: str, right: str) -> bool: def alternatives(matches) -> list[tuple[str, ...]]: - """Matched ids, grouped where one text cannot tell them apart. + """Group tied matches that their texts cannot distinguish. - Family members that tie survive `_dominated` together: GPL-3.0-only and - GPL-3.0-or-later share one reference text. `_name_for` says which id, if - any, such a group names. + For example, GPL-3.0-only and GPL-3.0-or-later share a reference text. """ entries = corpus()[1] groups: list[list[str]] = [] @@ -447,12 +401,10 @@ def alternatives(matches) -> list[tuple[str, ...]]: def _name_for(group: tuple[str, ...]) -> str | None: - """The id a text matching this group of `alternatives` names, if any. + """Select an unsuffixed base ID when every alternative extends it. - Where every other id extends one by a suffix -- MPL-2.0 and its - no-copyleft-exception variant, OFL-1.1 and its -RFN and -no-RFN -- that one - is SPDX's id for the unmodified text, and a notice selects the variant. - -only and -or-later have no such id: naming either asserts a choice. + MPL/OFL variants need a selecting notice; GPL -only/-or-later has no base + ID, so choosing either from the text alone would assert an unsupported grant. """ plain = [ base @@ -463,15 +415,14 @@ def _name_for(group: tuple[str, ...]) -> str | None: def _either(names) -> str: - """One text matching references it cannot choose between, for a reader.""" return "matches the reference text of " + " or ".join(names) def matched_prose(matches, name=lambda match: match[0], join=" AND ", lead=""): - """What a text matched, for a reader: a tie is one text matching either - reference, never both licences at once. `name` renders one match, and - `lead` precedes the matches named outright, joined by `join`; a prose - clause is set off with "; " so no " AND " ever reads as joining it.""" + """Render matches with tied alternatives as prose, not a license conjunction. + + Separate alternative clauses with semicolons so AND cannot join them. + """ if not matches: return None by_id = {match[0]: match for match in matches} @@ -502,17 +453,13 @@ def _read(path: Path) -> str | None: def evidence_for( component: str, origin: str, display: str, text: str, kind: str ) -> LicenseEvidence: - # A reference text is not matched: scoring it against the corpus it came - # from returns a perfect score, which reads as a measurement of the - # component's own file and is a measurement of nothing. Nested terms are - # matched to tell their holders from the licence's steward, never to - # conclude the component's licence. + # Do not score reference texts against their own corpus. Nested matches + # identify license-author notices; they do not conclude the component license. matchable = kind in ("grant", "nested") and origin != SPDX_REFERENCE tag = _SPDX_TAG.search(text[:4096]) if matchable else None matches = tuple(identify(text)) if matchable else () identified = matched_prose(matches) if origin == SPDX_REFERENCE: - # Named by the identifier it was fetched for, with no score behind it. identified = Path(display).stem return LicenseEvidence( component=component, @@ -529,13 +476,11 @@ def evidence_for( def files_in(directory: Path) -> Callable[[str], bool]: - """Whether `directory` holds a file of a given name: `is_sidecar`'s `beside`.""" return lambda name: (directory / name).is_file() def is_sidecar(name: str, beside: Callable[[str], bool]) -> bool: - """A REUSE `.license` sidecar: the annotation of a file `beside` finds next - to it. Named so with nothing to annotate, it is a licence file.""" + """A .license file is a sidecar only when its annotated file exists beside it.""" annotated = name.removesuffix(".license") return annotated not in {name, ""} and beside(annotated) @@ -553,7 +498,7 @@ def classify(name: str, beside: Callable[[str], bool]) -> str | None: def discover_in_tree( component: str, root: Path, display_prefix: str ) -> list[LicenseEvidence]: - """Every license and notice file a source tree carries, at its root or in LICENSES/.""" + """Collect root grants/notices, reference pools, and scoped nested terms.""" found: list[LicenseEvidence] = [] candidates: list[tuple[Path, str]] = [] @@ -565,11 +510,8 @@ def discover_in_tree( (item, "pool") for item in sorted(entry.rglob("*")) if item.is_file() ) - # A subdirectory can carry its own terms and its own holder -- a vendored - # asset set inside a larger project. Collected as "nested" so it neither - # discharges the component's packaging obligation nor changes its licence, - # and kept only where what it covers reached the wheel; see - # `build._with_reached_nested`. + # Nested terms apply only to reached files (build._with_reached_nested); + # they neither satisfy the root grant requirement nor set the component license. for path in sorted(root.rglob("*")) if root.is_dir() else []: if ( path.is_file() @@ -598,7 +540,6 @@ def discover_in_tree( scope = f"{display_prefix}/{covered.relative_to(root).as_posix()}" item = replace(item, scope=scope) found.append(item) - # "nested" never satisfies the grant test below; that is the point. if not grant_texts(found): found.extend(_readme_fallback(component, root, display_prefix)) @@ -622,13 +563,8 @@ def _readme_fallback( section = _README_LICENSE_SECTION.search(text) if section is None or not section.group("body").strip(): continue - # The section, not the whole README. Hashing the file would publish its - # build instructions as the terms that shipped, and would bury a real - # licence quoted there under enough prose to fall below the coverage - # threshold and degrade to a LicenseRef. - # The fragment is load-bearing: the digest below is of the section, not - # of the file, and a path naming the whole file beside it is a pair a - # verifier cannot reproduce. + # Hash and package only the licensing section. The path fragment makes its + # digest reproducible and keeps unrelated README prose out of the terms. display = f"{display_prefix}/{entry.name}#license-section" return [ evidence_for( @@ -653,12 +589,10 @@ def license_ref(component_key: str) -> str: def expressible(license_id: str) -> bool: - """Whether the SPDX grammar we ship can validate this identifier. + """Check identifiers against the pinned expression grammar. - The SPDX license list the build fetches moves ahead of the list bundled in - `license-expression`, so the corpus can name an ID the grammar cannot parse. - Matching against the newer corpus is right; naming an ID that a consumer's - validator -- which ships the older list -- will reject is not. + The fetched SPDX list may be newer than license-expression's bundled list; + newer IDs must travel as LicenseRefs until the grammar supports them. """ if license_id.startswith("LicenseRef-"): return True @@ -669,7 +603,6 @@ def expressible(license_id: str) -> bool: def grammar_names(identifiers) -> bool: - """Whether the SPDX grammar this document ships with can name every id.""" return all(expressible(item) or is_exception(item) for item in identifiers) @@ -687,7 +620,6 @@ def _explained(text: str, groups) -> float: def _without_notices(text: str) -> str: - """The text without the lines `read_notices` takes its notices from.""" notices = [_normalize(notice) for notice in read_notices(text)] return "\n".join( line @@ -715,7 +647,7 @@ def why_unnamed(evidence) -> str: for group in sorted({group for grant in carried for group in grant.tied}) ] reasons += [ - # Rounded down: "only 75%" beside a 75% threshold reads as a contradiction. + # Round down so a below-threshold result cannot display as meeting the threshold. "what matched in the SPDX reference corpus accounts for only " f"{math.floor(grant.explained * 100)}% of {grant.text.path}, its " "copyright notices aside" @@ -737,11 +669,8 @@ def why_unnamed(evidence) -> str: def why_unresolved(evidence) -> str: - """Why a component whose texts shipped concludes NOASSERTION. - - Without a grant, its only texts are a REUSE pool. With one, its grants - yielded no id: the grammar refused to combine them, or an SPDX reference - text carries no identified id. + """Explain NOASSERTION: either a reference pool without a grant, or + grants whose identifiers could not form an expression. """ if grants := grant_texts(evidence): paths = ", ".join(sorted(item.path for item in grants)) @@ -760,7 +689,6 @@ def _listed() -> _Listed: def listed_exception(identifier: str) -> bool: - """Whether the pinned list names this id as an exception.""" return identifier in _listed().exceptions @@ -783,12 +711,10 @@ def exception_evidence(component_key: str, expression: str) -> list[LicenseEvide def why_unlicensed(expressions) -> str | None: - """Why one vendored file's REUSE expressions fail the per-file gate, if they do. + """Explain a vendored file's REUSE declaration failure, if any. - `expressions` are REUSE's `SpdxExpression`s, so REUSE decides what parses. - Every licence named needs a reference text: a LicenseRef beside MIT would - otherwise be defined in the document by MIT's text, and an obligation with - none ships nothing. An exception needs only to be listed as an exception. + Every license operand needs its own reference text: MIT cannot supply the + terms for an adjacent LicenseRef. Exceptions need a listed exception ID. """ entries = corpus()[1] licences: set[str] = set() @@ -850,9 +776,7 @@ def why_unlicensed(expressions) -> str | None: @dataclass(frozen=True) class _Grant: - """One grant as read: the ids it names, split by whether the grammar can - express them, the tied groups it names none for, and how much of it its - matches account for, a tie counted once.""" + """A grant's supported/newer IDs, tied alternatives, and text coverage.""" text: LicenseEvidence named: tuple[str, ...] = () @@ -884,8 +808,7 @@ class _Reading: def _read_grant(item: LicenseEvidence) -> _Grant: if item.origin == SPDX_REFERENCE: - # Named by the identifier it was fetched for; matching it against the - # corpus it came from would be circular. + # The declared ID identifies this text; matching it to its own corpus is circular. ids, tied, explained = [item.identified] if item.identified else [], (), 1.0 else: groups = alternatives(item.matches) @@ -903,11 +826,11 @@ def _read_grant(item: LicenseEvidence) -> _Grant: def _reading(evidence) -> _Reading: - """Every grant names the ids it can and is carried verbatim where its ids - do not say everything: it matches nothing, ties without a plain id, is - mostly unmatched, or needs an id the grammar cannot express. A REUSE - declaration keeps its structure, an OR or a WITH, so one such id carries - every text it declares.""" + """Separate named grants from terms that must travel verbatim. + + If a REUSE declaration contains an unsupported ID, carry all of its texts + together to preserve OR/WITH semantics. + """ read = [_read_grant(item) for item in grant_texts(evidence)] whole = any(grant.declared and grant.newer for grant in read) carried = [ @@ -933,13 +856,10 @@ def names(grant: _Grant) -> tuple[str, ...]: def declared_terms( component_key: str, expressions ) -> tuple[str, list[LicenseEvidence]]: - """A vendored component's terms as its REUSE info declares them, and the - reference texts that convey them. + """Preserve REUSE expressions and supply their reference texts. - The expressions keep their structure -- rebuilding them from the texts - turns `A OR B` into `A AND B`. Where `_reading` carries a text, the whole - declaration travels as the component's LicenseRef, its exceptions' texts - with it. + Reconstructing from texts would turn OR into AND. Unsupported declarations + travel whole as a LicenseRef, including exception texts. """ declared = combine(sorted(expressions)) evidence = canonical_evidence(component_key, set(expressions)) @@ -952,28 +872,16 @@ def declared_terms( def expression(evidence: list[LicenseEvidence], component_key: str) -> tuple[str, str]: - """Concluded and declared expressions for one component. - - Concluded names what was matched in a text that ships with the artifact. A - grant `_reading` carries is not dropped and not guessed at: it adds the - component's LicenseRef, whose text travels in the document, which says - "these are the terms we shipped" without claiming to know which license - they are. - """ + """Conclude matched IDs plus a LicenseRef for any unresolved grant text.""" reading = _reading(evidence) identified = reading.named + ( [license_ref(component_key)] if reading.carried else [] ) concluded = _combine(identified) if identified else NOASSERTION - # Declared is the same expression, because the text it was read from is the - # component's own LICENSE -- shipping that file is how authors declare a - # licence, so one artifact answers both questions. - # - # Never from a REUSE tag on a grant file. OpenXR's COPYING.adoc is prose - # about which licences the project uses, tagged CC-BY-4.0 for the prose, and - # promoting that declared a linked Apache-2.0 SDK under a documentation - # licence. + # Declared and concluded both derive from the component's grant text. + # Do not use its REUSE tag: OpenXR's COPYING.adoc is tagged CC-BY-4.0 + # for the documentation, not for the SDK it describes. return concluded, concluded @@ -990,13 +898,7 @@ def _combine(expressions: list[str]) -> str: def normalized_expression(expression: str) -> str: - """Put an expression through the SPDX grammar before it reaches a document. - - Upstream tags are written by hand and arrive with lowercase operators and - odd spacing; emitting those unchanged would put an unparseable expression in - an SBOM. Anything the grammar rejects is reported as NOASSERTION rather than - passed through. - """ + """Normalize an SPDX expression, returning NOASSERTION when parsing fails.""" try: return str(_spdx_licensing().parse(expression, validate=False)) except (ExpressionError, ValueError, TypeError): @@ -1013,13 +915,9 @@ def read_spdx_tag(text: str) -> str | None: def fold_notices(notices) -> list[str]: - """Drop a notice another already covers. + """Remove notices subsumed by another notice for the same holder. - Files written in different years state one holder over different year ranges, - and listing every one names that holder several times over. Merging the - ranges instead would claim a year no file claims, so only a notice whose - holder matches another's and whose years that other already contains is - dropped -- it says nothing the one kept does not. + Do not merge disjoint year ranges: that would claim years no input states. """ parsed = [] for notice in notices: @@ -1040,20 +938,17 @@ def fold_notices(notices) -> list[str]: ) -# A notice is `Copyright` plus a marker that prose does not carry: a (c), a ©, -# or a year. It need not start the line and the lead-in is not bounded: a NOTICE -# writes `Box collision code (engine_collision_box.c) is Copyright 2016 ...`. +# Require a copyright marker or year, allowing an unbounded prose lead-in +# such as "Box collision code (...) is Copyright 2016 ...". _PLAIN_COPYRIGHT = re.compile( r"^.*?(?PCopyright\b\s*(?:\(c\)|©|\d{4})[^\n\r]*)$", re.MULTILINE | re.IGNORECASE, ) -# A clause broken by the end of a line, left dangling by taking the line alone. -# Words a notice can end on only because the line broke mid-clause. +# Connectives that indicate a notice was cut off mid-clause. _DANGLING_WORDS = frozenset( {"and", "or", "is", "was", "are", "were", "by", "under", "the", "a"} ) -# Years, year ranges and the markers around them: what is left is the holder. -# A sentence, as against a name: the Debian `Copyright:` field holds both. +# Distinguish explanatory prose from holder names in Debian Copyright fields. _PROSE = re.compile( r"\b(derived|contributed|reserved|portions?|redistribut\w+|written|" r"modified|see|provided|permission|granted|warrant\w*|software|hereby|" @@ -1081,22 +976,17 @@ def normalized_tag(text: str) -> str | None: # A LicenseRef here has no text to define it with, as for any other file. if expression == NOASSERTION or license_refs(expression): return None - # And the grammar has to know every id in it. `normalized_expression` parses - # without validating, so a tag naming nothing real reached the document and - # failed its own conformance check -- the same reason a conclusion is tested. + # Parsing without validation accepts unknown IDs; validate before emitting. if not all(map(expressible, _spdx_licensing().license_keys(expression))): return None return expression def notice_is_the_licence_authors(notice: str, identified: list[str]) -> bool: - """Whether this notice belongs to the licence rather than to the component. + """Identify notices belonging to the license author, not the component. - A COPYING that *is* the reference text carries the steward's own line -- the - FSF's in every GPL, Sam Hocevar's in WTFPL -- so a component shipping one - verbatim would be credited to whoever wrote the licence. The corpus already - holds those texts, so the question is answered by looking rather than by - keeping a list of stewards. + Compare against the corpus rather than maintaining a steward list + (for example FSF in GPL, or Sam Hocevar in WTFPL). """ entries = corpus()[1] collapsed = _steward_form(notice) @@ -1115,13 +1005,7 @@ def _steward_form(text: str) -> str: def read_notices(text: str) -> list[str]: - """Every concrete copyright notice a licence text states. - - All of them: glfw names two holders and miniz names two, and under MIT, BSD - and Zlib the notice is the obligation, so keeping the first is a notice file - that does not discharge it. A reference text carries the template instead -- - `Copyright [yyyy] [name of copyright owner]` -- which names nobody. - """ + """Read all concrete copyright notices, excluding unfilled reference templates.""" lines = text.splitlines() found: list[str] = _debian_copyright_fields(lines) for index, line in enumerate(lines): @@ -1129,10 +1013,7 @@ def read_notices(text: str) -> list[str]: if not match: continue notice = " ".join(match.group("notice").split()).rstrip("*/ ") - # The holder can sit on the lines below: after a trailing comma (libccd - # names a department, a faculty and a university), or after nothing at - # all (Debian copyright files and qhull put the years on one line and the - # holder on the next). Keep reading until a holder appears. + # Holders may continue below a year-only line or trailing comma (Debian, qhull, libccd). step = index while step + 1 < len(lines) and step - index < 3: if not (notice.endswith(",") or not _names_a_holder(notice)): @@ -1145,9 +1026,7 @@ def read_notices(text: str) -> list[str]: if ahead >= len(lines): break following = " ".join(lines[ahead].split()) - # A new notice, a field of its own, or a sentence is not a holder: - # a year-only notice followed by the licence body would otherwise - # take its first line as the name. + # Stop before a new notice, field, or license clause is mistaken for a holder. if ( not following or _PLAIN_COPYRIGHT.match(following) @@ -1200,20 +1079,13 @@ def _copyright_fields(lines: list[str]) -> list[list[list[str]]]: def _debian_copyright_fields(lines: list[str]) -> list[str]: - """Holders a Debian copyright file states without the word `Copyright`. - - `Copyright:` is a field there, and its value may be a bare name -- libmd - credits Poul-Henning Kamp, Colin Plumb and Steve Reid that way. Nothing else - in the file names them, so a parser looking for the word finds nobody. - """ + """Read bare holders from Debian Copyright fields without requiring the word Copyright.""" found: list[str] = [] for field in _copyright_fields(lines): values = [value for paragraph in field for value in paragraph] for position, value in enumerate(values): holder = value.rstrip(".") - # A field whose value opens with a notice proper is read by the scan - # below, prose and all; only a field naming a bare holder is read - # here, and only for as long as it keeps naming one. + # Full notices are handled by read_notices; stop here when bare holders end. if _PLAIN_COPYRIGHT.match(holder) or _PROSE.search(holder): break if position and len(holder) > 60: @@ -1224,13 +1096,9 @@ def _debian_copyright_fields(lines: list[str]) -> list[str]: def dep5_copyright(text: str) -> list[str] | None: - """The notices in the `Copyright:` fields of a machine-readable Debian - copyright file; None where the file is not one (no `Format:` field opening - it). + """Read DEP5 Copyright fields, or return None without an opening Format field. - Only the fields are read: licence paragraphs quote other people's notices. - Stanzas covering only the Debian packaging name its packagers, not the - library's holders. + Ignore quoted notices in license prose and packaging-only stanzas. """ lines = text.splitlines() first = next((line for line in lines if line.strip()), "") @@ -1293,7 +1161,6 @@ def opens(line: str, current: str) -> bool: def _stanzas(lines: list[str]) -> list[list[str]]: - """A Debian control-style file's stanzas, split at blank lines.""" stanzas: list[list[str]] = [[]] for line in lines: if line.strip(): @@ -1304,10 +1171,9 @@ def _stanzas(lines: list[str]) -> list[list[str]]: def _packaging_only(stanza: list[str]) -> bool: - """Whether a stanza's `Files:` patterns all lie under `debian/`. + """Identify stanzas limited to debian/; their holders are packagers. - Those name whoever packaged the library for Debian, not its holders. A - stanza with no `Files:` field -- the header -- is not one. + A header without Files is not packaging-only. """ patterns: list[str] = [] for index, line in enumerate(stanza): @@ -1336,7 +1202,6 @@ def _without_dangling(notice: str) -> str: def _names_a_holder(notice: str) -> bool: - """Whether anything is left once the years and their punctuation come out.""" rest = _YEARS_AND_MARKS.sub(" ", notice[len("Copyright") :]) return bool(re.search(r"[A-Za-z]{2}", rest)) @@ -1348,19 +1213,15 @@ def read_copyright(text: str) -> str | None: def combine(expressions: list[str]) -> str: - """Public form of the AND-combiner, for callers assembling an expression.""" return _combine(expressions) def canonical_evidence( component_key: str, expressions: set[str] ) -> list[LicenseEvidence]: - """The SPDX reference text for the ids a vendored file's REUSE info declares. + """Fetch reference texts for vendored declarations from the pinned SPDX list. - The text its identifier names is what conveys those terms. It comes from the - list this build pinned and fetched -- never from this repository's own REUSE - pool, whose copies exist to license *this* project and carry its notices, - not the component's. + Do not use the repository's REUSE pool: its texts can carry our own notices. """ found: list[LicenseEvidence] = [] seen: set[str] = set() @@ -1369,10 +1230,7 @@ def canonical_evidence( try: parsed = _spdx_licensing().parse(expression, validate=False) except (ExpressionError, ValueError, TypeError): - # REUSE info is written by hand and can say anything -- "Public - # domain or MIT License" is not an expression. One that will not - # parse names no reference text; the build's per-file gate names the - # file that states it, so nothing stops here. + # The per-file build gate reports malformed declarations with their source paths. continue for token in _spdx_licensing().license_keys(parsed): reference = entries.get(token) @@ -1396,13 +1254,9 @@ class ExpressionReadError(Exception): def shared_refs(components) -> dict[str, str]: - """Rename map putting components with byte-identical terms on one LicenseRef. + """Share a LicenseRef among components with byte-identical terms. - A LicenseRef identifies a text, not a component, so two builds of one SDK - under the same EULA share an id. The map is produced here, before anything - renders, so the document, the report and the packaged notices all name the - same id -- they are cross-referenced, and a text shipped under one name in - the wheel's notices and another in the wheel's SBOM reconciles with nothing. + Resolve IDs before rendering so notices, reports, and SPDX agree. """ by_text: dict[str, list[str]] = {} for component in components: @@ -1424,7 +1278,6 @@ def shared_refs(components) -> dict[str, str]: def shared_ref(text: str) -> str: - """The id components carrying byte-identical terms share, named by them.""" return SHARED_REF + sha256_text(text)[:12] @@ -1433,11 +1286,7 @@ def sha256_text(text: str) -> str: def rename_refs(expression: str, renamed: dict[str, str]) -> str: - """Substitute whole identifiers, never substrings. - - `LicenseRef-alpha` is a prefix of `LicenseRef-alpha-extra`, so a plain - replace turns the longer id into one nothing defines and orphans its entry. - """ + """Replace whole identifiers: LicenseRef-alpha must not alter LicenseRef-alpha-extra.""" for old_id, new_id in renamed.items(): expression = re.sub( rf"(? str: def is_exception(identifier: str) -> bool: - """Whether an identifier is a licence exception rather than a licence. - - An exception modifies the licence it is attached to; the pinned list ships - their texts separately, and one ships only where its expression travels as a - LicenseRef. - """ + """Check whether an identifier names an SPDX license exception.""" try: parsed = _spdx_licensing().parse(identifier, validate=False) except (ExpressionError, ValueError, TypeError): @@ -1462,11 +1306,9 @@ def is_exception(identifier: str) -> bool: def operands_of(expression: str) -> list[str]: - """An expression split into parts that may be AND-joined without changing it. + """Split expressions into operands that can be AND-joined without changing meaning. - An `OR` is one choice and splitting it asserts both halves, so it stays - whole. Everything else decomposes to identifiers, which keeps a roll-up flat - and free of an id two components happen to share. + Keep OR choices whole; splitting them would require both alternatives. """ try: parsed = _spdx_licensing().parse(expression, validate=False) @@ -1493,9 +1335,7 @@ def license_refs(*expressions: str) -> list[str]: try: parsed = _spdx_licensing().parse(expression, validate=False) except (ExpressionError, ValueError, TypeError) as error: - # Walking past it hid every reference inside: a document could carry - # `LicenseRef-never-defined AND (((` and be reported as defining - # everything it uses. + # Malformed expressions must fail closed; skipping one could hide undefined references. raise ExpressionReadError(f"{expression!r}: {error}") from error found.update( key diff --git a/scripts/sbom/isaaccapture_sbom/report.py b/scripts/sbom/isaaccapture_sbom/report.py index 5618eb621a..d747a27b67 100644 --- a/scripts/sbom/isaaccapture_sbom/report.py +++ b/scripts/sbom/isaaccapture_sbom/report.py @@ -26,9 +26,7 @@ } _SAFE_NAME = re.compile(r"[^A-Za-z0-9._-]") -# What a row is, so a copyright holder is not read as a package name. A vendored -# row is named for the holder its REUSE info or header names, the only identity -# such material has -- there is no package behind it. +# Distinguish vendored copyright holders from installable packages. _KIND_PROSE = { "source-tree": "fetched source", "archive": "fetched archive", @@ -40,9 +38,7 @@ _COMMIT = re.compile(r"^[0-9a-f]{40}$") -# Every list of components whose terms a reader cannot take as identified, with -# its Markdown heading. The report, its Markdown, the manifest and the CI -# warnings all read this, so a list added here cannot be left out of one. +# Shared gap categories for JSON, Markdown, manifests, and CI warnings. GAP_LISTS = { "components_without_license_evidence": "No license text found for:", "components_with_unresolved_license": "No license concluded for:", @@ -51,11 +47,7 @@ def _short(version: str) -> str: - """Abbreviate a commit; leave anything else whole. - - Truncating to a fixed width turned `1.1.0-2build1.1` into `1.1.0-2build`, - which reads as a complete Debian version and is not one. - """ + """Abbreviate commit hashes only; truncating a package version changes its identity.""" return version[:12] if _COMMIT.match(version) else version @@ -65,7 +57,6 @@ def licenses_root(dist_info: str) -> str: def notices_path(dist_info: str) -> str: - """The notice file that travels inside the wheel, by name.""" return f"{licenses_root(dist_info)}THIRD-PARTY-NOTICES.md" @@ -74,36 +65,23 @@ def is_packaged_license(dist_info: str, member: str) -> bool: def is_third_party_license(dist_info: str, member: str) -> bool: - """A text read out of a component, as against this distribution's own.""" return member.startswith(f"{licenses_root(dist_info)}third-party/") def declared_license_path(dist_info: str, member: str) -> str: - """What METADATA calls a packaged licence file. - - PEP 639 states License-File relative to `.dist-info/licenses/`, which is - what setuptools already does for the project's own texts; declaring the - member path instead yields an entry that resolves to nothing. - """ + """PEP 639 License-File paths are relative to .dist-info/licenses/.""" return member.removeprefix(licenses_root(dist_info)) -# What a nested text is, wherever one is listed. Some restate the component's -# own grant, so this says only where they apply. +# Nested texts may repeat the root grant; describe their scope without implying a difference. NESTED_ROLE = "terms stated for part of this component" def packaged_license_path(dist_info: str, component_key: str, evidence) -> str: - """Where one component's license text lands inside the wheel. - - Named after the evidence's whole path within its component, not its - basename: a REUSE pool makes `LICENSE` and `LICENSES/MIT.txt` collide, and - dropping one would leave the notices pointing at a text that is not there. - """ + """Place terms under the component's full evidence path to avoid basename collisions.""" folder = _SAFE_NAME.sub("_", component_key) relative = evidence.path.rsplit("!", 1)[-1].split("/", 1)[-1] - # Keep the separators: flattening them re-creates the collision this scheme - # exists to avoid, and the document would name a text that is not there. + # Preserve path separators when sanitizing to avoid flattening distinct paths. safe = "/".join(_SAFE_NAME.sub("_", part) for part in relative.split("/")) return f"{licenses_root(dist_info)}third-party/{folder}/{safe}" @@ -119,16 +97,12 @@ def notices_markdown( wheel_name: str, project: dict | None = None, ) -> str: - """The notice file that travels inside the wheel. + """Render notices with the distribution's own license and copyright first. - It opens with the distribution's own licence and holder. The reference text - a declared licence packages names nobody -- Apache-2.0's appendix is a - template -- so for a wheel shipping no source file of its own this is the - only place a recipient can read who licensed it to them. + Reference templates do not name the holder of a metadata-only wheel. """ lines = ["# Notices", ""] if project: - # `Organization: NVIDIA` is how SPDX spells a supplier; prose does not. supplier = project["supplier"].split(": ", 1)[-1] lines += [f"`{wheel_name}` is distributed by {supplier}."] if project.get("license", NOASSERTION) != NOASSERTION: @@ -138,8 +112,6 @@ def notices_markdown( lines.append("") lines += ["## Third-party notices", ""] if not inventory.components_present: - # Saying where texts live when none were written sends a reader to a - # directory the wheel does not contain. lines += [ f"`{wheel_name}` redistributes no third-party component, so no " "license text is packaged here.", @@ -158,11 +130,8 @@ def notices_markdown( lines.append("") lines.append(f"- License: `{component.license_concluded}`") if component.copyright_text != "NOASSERTION": - # A reference text names no holder, so for a component that ships no - # licence file of its own this is the only notice in the wheel. - # Joined with an explicit break: a newline inside a list item is a - # lazy continuation, and so is an indented one, so either way a - # renderer runs distinct legal notices into one paragraph. + # Reference texts omit component holders. Render their notices separately, + # with explicit breaks so Markdown does not merge them into one paragraph. lines.append( "- Copyright: " + "
".join(component.copyright_text.splitlines()) ) @@ -248,9 +217,6 @@ def report(inventory: Inventory, evidence_doc: dict, wheel_name: str) -> dict: "evidence": [item.as_json() for item in component.evidence], } ) - # Same test as the build gate and the verifier: a REUSE pool is a real - # text. Requiring a grant here advertised "no licence evidence" for a - # component the gates deliberately accepted on the pool they packaged. if not obligations: gaps["components_without_license_evidence"].append( { @@ -259,8 +225,7 @@ def report(inventory: Inventory, evidence_doc: dict, wheel_name: str) -> dict: } ) elif component.license_concluded == NOASSERTION: - # Texts that pass the gate can still name nothing - # (`licensing.why_unresolved`). + # Packaged text can satisfy the gate without identifying a license. gaps["components_with_unresolved_license"].append( { "component": key, @@ -269,10 +234,7 @@ def report(inventory: Inventory, evidence_doc: dict, wheel_name: str) -> dict: } ) elif carried := licensing.carried_texts(component.evidence): - # Not whether anything matched: a text can match and still be - # carried (`licensing.carried_texts`), and one beside it can match - # and be named. Either way these terms are only readable as the - # text this wheel carries. + # A partially matched grant still carries unresolved terms. gaps["components_with_unidentified_license"].append( { "component": key, @@ -285,13 +247,11 @@ def report(inventory: Inventory, evidence_doc: dict, wheel_name: str) -> dict: "schema": "isaaccapture-license-report/2", "wheel": wheel_name, "generated_from_commit": evidence_doc.get("project", {}).get("commit"), - # The commit alone does not describe a tree that was modified, and this - # report is what a reader uses to reproduce the build. "source_tree_modified": str( evidence_doc.get("project", {}).get("describe") or "" ).endswith("-dirty"), "spdx_license_list_version": evidence_doc.get("license_list_version"), - # Where the build read the list from is the build evidence's to record. + # Keep builder-local corpus paths in build evidence only. "license_data": { key: value for key, value in evidence_doc.get("license_data", {}).items() @@ -331,11 +291,7 @@ def report_markdown(payload: dict) -> str: "| --- | --- | --- | --- | --- | --- |", ] for item in payload["components"]: - # Grants in full; the REUSE pool summarised. A pool is a directory of - # reference texts kept so that per-file tags resolve -- OpenXR's covers - # its fonts and spec prose -- so listing all of it buries the two files - # the licence was actually read from. Every path is in the JSON report - # and in the packaged notices, which is where the paths matter. + # Summarize reference pools for readability; JSON and packaged notices retain all paths. shown = [record for record in item["evidence"] if record["kind"] != "pool"] pooled = [record for record in item["evidence"] if record["kind"] == "pool"] parts = [ diff --git a/scripts/sbom/isaaccapture_sbom/validate.py b/scripts/sbom/isaaccapture_sbom/validate.py index c15fa8612b..eb7288e7a2 100644 --- a/scripts/sbom/isaaccapture_sbom/validate.py +++ b/scripts/sbom/isaaccapture_sbom/validate.py @@ -1,10 +1,9 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -"""Checks that decide whether a wheel may be published. +"""Verify wheel contents and optional build/release evidence. -`verify_wheel` is the consumer-side half: it needs the wheel and nothing else, -so the instructions published with a release are the same code that gates it. +verify_wheel needs only the archive and is shared by CI and consumers. """ from __future__ import annotations @@ -28,11 +27,10 @@ class ManifestError(Exception): - """A manifest is not shaped the way this tool writes them.""" + """A manifest lacks the structure required for verification.""" def _sidecar(evidence_dir: Path, record: dict, kind: str) -> Path | None: - """A named sidecar, or None so the missing-file failure is reported as one.""" named = record.get("sidecars", {}).get(kind, {}).get("filename") if not named: return None @@ -66,7 +64,6 @@ def manifest_records(path: Path) -> list[dict]: def bound_names(records) -> list[str]: - """The sidecar file names the records bind.""" return [ sidecar["filename"] for record in records @@ -75,7 +72,7 @@ def bound_names(records) -> list[str]: def sidecar_failures(beside: Path, record: dict) -> list[str]: - """Each sidecar a checked record binds that is not beside it as bound.""" + """Return missing-file or digest failures for a checked record's sidecars.""" failures = [] for kind, sidecar in record["sidecars"].items(): path = beside / sidecar["filename"] @@ -92,7 +89,7 @@ def _is_bare_name(name: str) -> bool: def _record_for(records: list[dict], filename: str) -> dict: - """One wheel's record, or a clear error rather than a KeyError traceback.""" + """Find exactly one record for a wheel, or raise ManifestError.""" seen = [item for item in records if item.get("filename") == filename] if len(seen) > 1: raise ManifestError(f"{filename} is advertised {len(seen)} times") @@ -121,9 +118,7 @@ def checked_record(record: dict) -> dict: f"{filename}: {kind} sidecar {item['filename']!r} is not a bare " "file name" ) - # Every sidecar, not just the ones a record happens to name. An absent entry - # is not an absent check: omitting `build_evidence` left `check-set` passing - # a wheel the evidence would have failed. + # Require every sidecar kind; omissions must not bypass evidence checks. missing = sorted(set(_SIDECAR_KINDS) - set(record["sidecars"])) if missing: raise ManifestError( @@ -171,10 +166,9 @@ def verify_wheel( scanned: wheelfile.WheelInfo | None = None, carried: list[dict] | None = None, ) -> list[str]: - """Verify a wheel against the SBOM it carries. No repository required. + """Verify an archive against its embedded SBOM without a repository. - `carried` collects the parsed document for callers that go on to check it - against evidence, so the wheel is opened and the document parsed once. + Append the parsed document to carried for reuse by evidence checks. """ failures: list[str] = [] wheel = scanned or wheelfile.scan(wheel_path) @@ -204,9 +198,7 @@ def verify_wheel( if duplicates: failures.append(f"duplicate SPDXIDs: {duplicates}") known = set(identifiers) - # An element in another document is written `DocumentRef-x:SPDXRef-y`, and - # the prefix has to be one this document declares -- otherwise it names a - # document no reader can fetch or check. + # Cross-document element IDs must name a declared DocumentRef. declared_documents = { item["externalDocumentId"] for item in spdx.get("externalDocumentRefs", []) } @@ -262,10 +254,8 @@ def verify_wheel( "packageVerificationCodeExcludedFiles", [] ) } - # Exactly these two, not "at least". The list lives in the document being - # checked, so treating it as a set of files to skip lets anyone who can edit - # the document exempt anything they add to the wheel from the coverage check - # below -- which is the whole of what this command promises a consumer. + # Do not trust the document to choose exclusions: arbitrary entries here + # could hide added files from coverage checks. permitted = {f"{wheel.dist_info}/RECORD", sbom_name} if excluded != permitted: failures.append( @@ -312,13 +302,9 @@ def verify_wheel( def _check_license_texts(wheel: wheelfile.WheelInfo, spdx: dict) -> list[str]: - """Every text the document describes must be a text the wheel carries. + """Check that cited license paths contain the bytes the document describes. - The document states a digest for each licence it read and carries the - verbatim text of each LicenseRef. The wheel packages those same bytes under - `licenses/`. Nothing compared the two, so a packaged licence could be - swapped for another -- or emptied -- and every per-file digest still agreed, - because those describe the replacement. + Per-file hashes alone can agree with a substituted or emptied license file. """ failures: list[str] = [] by_name = {entry.name: entry.sha256 for entry in wheel.entries} @@ -335,9 +321,7 @@ def _check_license_texts(wheel: wheelfile.WheelInfo, spdx: dict) -> list[str]: match = _EVIDENCE_DIGEST.search(text) if not match: continue - # The path matters, not just the bytes existing somewhere: two - # components can ship an identical text, so a swap inside one of them - # survives a check that only asks whether the digest is present. + # Check the cited path too: another component can contain identical license bytes. where = match.group("where") if by_name.get(where) != match.group("digest"): failures.append( @@ -349,7 +333,6 @@ def _check_license_texts(wheel: wheelfile.WheelInfo, spdx: dict) -> list[str]: def _check_record(wheel_path: Path, wheel: wheelfile.WheelInfo) -> list[str]: - """RECORD is what an installer trusts; a wrong entry here is a broken wheel.""" failures: list[str] = [] record = wheelfile.read_record(wheel_path, wheel.dist_info) record_name = f"{wheel.dist_info}/RECORD" @@ -360,8 +343,7 @@ def _check_record(wheel_path: Path, wheel: wheelfile.WheelInfo) -> list[str]: if row is None: failures.append(f"RECORD does not list {entry.name}") continue - # scan() already hashed every member; RECORD states the same digest in - # base64, so re-reading the archive to recompute it is wasted work. + # Reuse scan hashes; RECORD encodes the same digest in base64. if row[0] != wheelfile.record_hash_from_sha256(entry.sha256): failures.append(f"RECORD digest mismatch for {entry.name}") if row[1] != str(entry.size): @@ -374,12 +356,9 @@ def _check_record(wheel_path: Path, wheel: wheelfile.WheelInfo) -> list[str]: def _check_attribution( evidence_doc: dict, wheel: wheelfile.WheelInfo, spdx: dict | None ) -> list[str]: - """Every wheel member must name where it came from, and every shipped - component must have had a license text to read.""" + """Require attribution for every member and packaged terms for each component.""" stated = evidence_doc.get("wheel") if stated and stated != wheel.path.name: - # Checking a wheel against another wheel's evidence compares two unrelated - # member sets; the mismatch it produces describes the wrong problem. return [f"build evidence describes {stated}, not {wheel.path.name}"] failures: list[str] = [] @@ -400,9 +379,7 @@ def _check_attribution( for key, component in evidence_doc.get("components", {}).items(): if not component.get("in_this_wheel"): continue - # Every kind: the document cites one attributionText per evidence item, - # so comparing only the licence-granting ones called a NOTICE the build - # plainly read a text it never did. + # Include NOTICE and other non-grant evidence cited by attributionTexts. recorded.update(item["sha256"] for item in component.get("evidence", [])) texts = [ item @@ -414,11 +391,7 @@ def _check_attribution( f"{key} is redistributed but no license text was found for it" ) continue - # Found is not shipped. The build gate asks whether a text existed to - # read; this asks whether the wheel in hand still carries it. Without - # the second, every statement the document makes about a component -- - # that it is here, under these terms, with this text -- could be deleted - # along with the text and nothing would notice. + # Existence at build time is insufficient; the recorded terms must still be packaged. if not any(item["sha256"] in packaged for item in texts): failures.append( f"{key} is redistributed but the license text recorded for it is " @@ -434,12 +407,10 @@ def _document_agrees_with_evidence( evidence_doc: dict, spdx: dict | None, ) -> list[str]: - """The texts the document names must be the ones the build actually read. + """Compare license claims with separately recorded build evidence. - `_check_license_texts` reads the expected digest out of the document it is - checking, so a rewrite consistent with itself is invisible to it. The build - evidence was produced before the wheel was published and states the digests - independently, so where it is supplied it is the thing to compare against. + A self-consistent rewrite of the SBOM and wheel can pass _check_license_texts; + the evidence supplies independent digests and conclusions. """ if spdx is None: return [] @@ -454,14 +425,10 @@ def _document_agrees_with_evidence( f"(sha256:{match.group('digest')}) the build never read" ) - # A digest is the one thing about a component that cannot change its terms. - # Comparing only that left every expression in the document rewritable -- - # a proprietary EULA could be published as Apache-2.0, with the evidence - # beside it still saying otherwise and every gate reporting OK. + # Compare expressions as well as text hashes; an unchanged EULA could otherwise + # be relabeled Apache-2.0. components = evidence_doc.get("components", {}) - # By the id the document assigns, not by display name: a source tree, a - # system library and an archive can all be called the same thing, and a name - # join silently drops all but one of them. + # Join by component ID: source trees, archives, and libraries can share display names. by_id = { document_module.component_package_id(key): (key, component) for key, component in components.items() @@ -477,10 +444,7 @@ def _document_agrees_with_evidence( ): expected = component.get(stated) actual = package.get(field) - # Exactly, LicenseRef ids included: the build settles shared ids - # before it renders either side, and the text check below ties a - # shared id to the text it names and a minted one to its owner, - # never either to the package naming it. + # Shared LicenseRefs are resolved before rendering, so require exact ID agreement. if not expected or actual == expected: continue failures.append( @@ -488,11 +452,8 @@ def _document_agrees_with_evidence( f"the build recorded {expected!r}" ) - # And the text a LicenseRef stands for has to be the terms the build read - # for some component. A component can state its licence across several - # files, so the body is a join with no digest of its own -- the evidence - # states that digest, which is what makes every entry checkable, shared ids - # included. An id minted for one component is held to that component's. + # Bind each LicenseRef to the combined terms recorded for its owner, or to + # a recorded shared text. Individual file hashes cannot verify a joined body. by_ref = {licensing.license_ref(key): key for key in components} stated = { key: component.get("verbatim_terms_sha256") @@ -501,13 +462,12 @@ def _document_agrees_with_evidence( any_stated = {digest for digest in stated.values() if digest} for extracted in spdx.get("hasExtractedLicensingInfos", []): ref, text = extracted["licenseId"], extracted.get("extractedText", "") - # A shared id names its text, so it is held to that text, not just to - # the terms some component has. + # A shared ID must match its text hash, not just any component's terms. if _SHARED_ID.fullmatch(ref) and ref != licensing.shared_ref(text): failures.append(f"{ref}: the text it carries is not the one its id names") continue if not any_stated: - # Evidence written before this digest existed cannot answer. + # Legacy evidence without this digest cannot verify the combined text. continue owner = by_ref.get(ref) expected = {stated[owner]} if owner and stated.get(owner) else any_stated @@ -520,11 +480,7 @@ def _document_agrees_with_evidence( def _refs_in(value) -> set[str]: - """LicenseRef identifiers in a field that may hold an expression or a list. - - An expression that cannot be read is reported as such rather than treated as - naming nothing. - """ + """Read LicenseRefs from an expression or list, failing on malformed expressions.""" if value is None: return set() items = value if isinstance(value, list) else [value] @@ -541,7 +497,6 @@ def _check_licenses(wheel: wheelfile.WheelInfo, spdx: dict) -> list[str]: if notices not in names: failures.append(f"{notices} is not packaged") - # Every field that can name one, through one parser. declared_refs = set() for item in spdx.get("files", []): for field in ("licenseConcluded", "licenseInfoInFiles"): @@ -549,7 +504,6 @@ def _check_licenses(wheel: wheelfile.WheelInfo, spdx: dict) -> list[str]: for package in spdx["packages"]: for field in ("licenseConcluded", "licenseDeclared", "licenseInfoFromFiles"): declared_refs.update(_refs_in(package.get(field))) - # Every file id a package claims to hold has to be a file this document has. file_ids = {item["SPDXID"] for item in spdx.get("files", [])} for package in spdx["packages"]: for held in package.get("hasFiles", []): @@ -575,12 +529,7 @@ def check( manifest: Path | None = None, evidence_dir: Path | None = None, ) -> list[str]: - """Check a wheel, and as much of its published evidence as you can supply. - - One operation over widening scope rather than three commands: a consumer has - only the wheel, a release build also has the evidence it just produced, and a - release has the whole set. Each input adds checks; none replaces the others. - """ + """Verify a wheel, adding checks for each supplied evidence or manifest input.""" scanned = wheelfile.scan(wheel) carried: list[dict] = [] failures = verify_wheel(wheel, scanned, carried) @@ -620,9 +569,6 @@ def _check_manifest( if published.is_file(): with zipfile.ZipFile(wheel) as archive: embedded = record["sbom_in_wheel"] - # Reported, not raised: the member name comes out of the manifest, - # so one the wheel does not hold is a disagreement between the two - # and reads as such, rather than as a KeyError from the zip reader. if embedded not in archive.namelist(): failures.append( f"the manifest names {embedded} as this wheel's SBOM, and the " @@ -638,10 +584,8 @@ def _check_manifest( def _cross_document_refs( spdx: dict, documents: dict[str, bytes], filename: str ) -> list[str]: - """An external document reference has to resolve to a document in this set. - - Declaring one is not the check: the digest is what binds it, and a wheel - rebuilt without its sibling would otherwise carry a stale one and pass. + """Resolve external references against this release, checking document hashes + and referenced elements to reject stale sibling references. """ failures: list[str] = [] by_namespace = { @@ -664,7 +608,6 @@ def _cross_document_refs( f"{filename}: {ref.get('externalDocumentId')} states sha1 {stated} " f"for {name}, which hashes to {actual}" ) - # And the elements those references point at have to be in that document. ids = { ref["externalDocumentId"]: json.loads(by_namespace[ref["spdxDocument"]][1]) for ref in spdx.get("externalDocumentRefs", []) @@ -688,8 +631,7 @@ def _cross_document_refs( def check_set(manifest_path: Path, wheel_dir: Path, evidence_dir: Path) -> list[str]: - """Run :func:`check` over every wheel a merged manifest advertises, and fail - any wheel or evidence file here that it does not name.""" + """Verify the release and reject wheels or evidence omitted from its manifest.""" failures: list[str] = [] records = manifest_records(manifest_path) for name, seen in sorted(Counter(item["filename"] for item in records).items()): @@ -697,14 +639,12 @@ def check_set(manifest_path: Path, wheel_dir: Path, evidence_dir: Path) -> list[ raise ManifestError(f"{name} is advertised {seen} times") by_name = {item["filename"]: checked_record(item) for item in records} - # Every wheel here is published with the set, so one the manifest omits - # would go out unchecked. + # Every downloaded wheel is published, so unlisted wheels cannot be left unchecked. for stray in sorted({item.name for item in wheel_dir.glob("*.whl")} - set(by_name)): failures.append( f"{stray} is in {wheel_dir}, and {manifest_path.name} does not list it" ) - # The same holds for evidence: a file no record binds is never verified. - # Wheels sharing the directory are the check above's to judge. + # Reject unbound evidence too; the wheel check above handles shared-directory wheels. expected = set(bound_names(by_name.values())) if manifest_path.parent.resolve() == evidence_dir.resolve(): expected.add(manifest_path.name) @@ -716,8 +656,6 @@ def check_set(manifest_path: Path, wheel_dir: Path, evidence_dir: Path) -> list[ f"{stray} is in {evidence_dir}, and {manifest_path.name} does not bind it" ) - # The whole release is here, which is the only place a reference from one - # document to another can be resolved rather than merely declared. published: dict[str, bytes] = {} for record in by_name.values(): sidecar = _sidecar(evidence_dir, record, "spdx") diff --git a/scripts/sbom/isaaccapture_sbom/wheelfile.py b/scripts/sbom/isaaccapture_sbom/wheelfile.py index bba2cf6f85..79d4eaecdb 100644 --- a/scripts/sbom/isaaccapture_sbom/wheelfile.py +++ b/scripts/sbom/isaaccapture_sbom/wheelfile.py @@ -1,11 +1,9 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -"""Read a wheel's contents and rewrite it with the evidence it must carry. +"""Read and rewrite wheels with packaged evidence. -RECORD and METADATA are specified formats, so their own tooling parses them: -``installer`` (PyPA) owns RECORD's hashing and escaping rules, and ``packaging`` -owns core metadata. Only the zip rewriting is ours, because no library offers it. +Use installer for RECORD parsing and packaging for core metadata. """ from __future__ import annotations @@ -30,18 +28,14 @@ from . import elf -# Fixed member timestamp for files this tool adds, so two runs over the same -# inputs produce byte-identical wheels. +# Deterministic default timestamp for added members. _ADDED_TIMESTAMP = (1980, 1, 1, 0, 0, 0) def _reproducible_timestamp() -> tuple[int, int, int, int, int, int] | None: - """SOURCE_DATE_EPOCH as a zip timestamp, when a reproducible build asked for one. + """Convert SOURCE_DATE_EPOCH to a ZIP timestamp. - Zip records a local timestamp per member, so a wheel carries whatever clock - its builder had. Under SOURCE_DATE_EPOCH every member gets the same stamp -- - including ones copied through untouched -- or the rewrite would be - reproducible only for an input wheel that already was. + Apply it to copied members too, so input timestamps cannot affect reproducibility. """ epoch = os.environ.get("SOURCE_DATE_EPOCH") if not epoch or not epoch.isdigit(): @@ -75,7 +69,7 @@ class Entry: needed: tuple[str, ...] = () machine: str | None = None build_id: str | None = None - # Enough to find a REUSE tag, kept so nothing re-opens the archive for it. + # Retain a prefix for REUSE tags to avoid reopening the archive. head: bytes = b"" @property @@ -93,11 +87,9 @@ class WheelInfo: def sbom_member(dist_info: str, wheel_name: str) -> str: - """Where this collector's SPDX document sits inside the wheel. + """Name this collector's document in the shared PEP 770 sboms directory. - PEP 770 makes `.dist-info/sboms/` a shared directory: auditwheel writes - `auditwheel.cdx.json` there for the libraries it vendors. Match this exact - name, never the directory, or a second tool's document reads as ours. + Do not select by directory alone: auditwheel.cdx.json may coexist there. """ return f"{dist_info}/sboms/{wheel_name}.spdx.json" @@ -111,7 +103,6 @@ def record_hash_from_sha256(digest: str) -> str: def record_hash(data: bytes) -> str: - """The same field, for bytes in hand.""" return record_hash_from_sha256(hashlib.sha256(data).hexdigest()) @@ -131,28 +122,19 @@ def _dist_info(names: list[str]) -> str: def dist_info_of(wheel_path: Path) -> str: - """The `.dist-info` directory name, from the archive's index alone. - - For a caller that wants only the metadata: `scan` hashes every member, which - over a release's worth of wheels is a full read of each for a name the - central directory already states. - """ + """Find .dist-info via the ZIP index without hashing every member.""" with zipfile.ZipFile(wheel_path) as archive: return _dist_info(archive.namelist()) def scan(wheel_path: Path, analyze_elf: bool = False) -> WheelInfo: - """Hash every member, and read the dynamic section of ELF members on request. + """Hash members, optionally reading ELF metadata for attribution. - `analyze_elf` is what the inventory needs and what verification does not, so - a consumer checking a downloaded wheel never runs a binary parser over it. + Consumer verification leaves analyze_elf off to avoid parsing binaries. """ with zipfile.ZipFile(wheel_path) as archive: names = archive.namelist() - # A zip may hold two members under one name. Reading by name gives the - # last of them, so the first is hashed by nothing, compared to RECORD by - # nothing, and reported as uncovered by nothing -- while which one a - # consumer extracts is up to their unzip. Refuse the archive. + # Duplicate names make reads and extraction ambiguous; reject them before hashing. repeated = sorted(item for item, seen in Counter(names).items() if seen > 1) if repeated: raise WheelError( @@ -173,7 +155,6 @@ def scan(wheel_path: Path, analyze_elf: bool = False) -> WheelInfo: for info in archive.infolist(): if info.is_dir(): continue - # By handle, not by name: the name is what was just proven ambiguous. with archive.open(info) as handle: data = handle.read() soname = needed = machine = build_id = None @@ -246,15 +227,12 @@ def add_license_files(raw_metadata: bytes, relative_paths: list[str]) -> bytes: if not additions: return raw_metadata - # email owns RFC 822 header insertion, including keeping the payload (the - # long description) intact. + # Use the email parser to preserve the long-description payload. message = message_from_bytes(raw_metadata, policy=compat32) for path in additions: message.add_header("License-File", path) - # maxheaderlen=0 disables RFC 822 folding. A folded value is still valid and - # packaging unfolds it, but no other tool wraps paths in METADATA, and a - # continuation line defeats anything reading it more simply. + # Disable header folding so License-File paths remain on one line. buffer = io.BytesIO() BytesGenerator(buffer, policy=compat32, maxheaderlen=0).flatten(message) return buffer.getvalue() @@ -266,8 +244,7 @@ def read_record(wheel_path: Path, dist_info: str) -> dict[str, tuple[str, str]]: lines = archive.read(f"{dist_info}/RECORD").decode("utf-8").splitlines() entries = (RecordEntry.from_elements(*row) for row in parse_record_file(lines)) return { - # `entry.size or ""` would erase a legitimate 0: py.typed and namespace - # __init__.py are empty, and RECORD states 0 for them. + # Preserve size 0 for empty files; `entry.size or ""` would erase it. entry.path: ( str(entry.hash_) if entry.hash_ else "", "" if entry.size is None else str(entry.size), @@ -289,11 +266,7 @@ def rewrite( replacements: dict[str, bytes], dist_info: str, ) -> None: - """Copy the wheel applying replacements and additions, then rebuild RECORD. - - RECORD is regenerated from what the new archive actually holds, so a wrong - digest cannot survive the step that writes it. - """ + """Rewrite members and regenerate RECORD from the resulting bytes.""" record_name = f"{dist_info}/RECORD" collisions = sorted( set(additions) & {item.filename for item in _members(wheel_path)} diff --git a/scripts/sbom/pyproject.toml b/scripts/sbom/pyproject.toml index 31c61b2db7..004a8d7b25 100644 --- a/scripts/sbom/pyproject.toml +++ b/scripts/sbom/pyproject.toml @@ -1,13 +1,7 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -# The wheel SBOM collector. A project of its own so its dependencies are -# declared once, here, rather than repeated at every call site. -# -# uv run --project scripts/sbom isaaccapture-sbom --help -# uv run --project scripts/sbom --extra dev pytest scripts/sbom/tests -# -# How it works: README.md here. What a wheel carries: docs/source/references/sbom.rst. +# Standalone collector project; usage and design are in README.md. [project] name = "isaaccapture-sbom" @@ -17,22 +11,18 @@ requires-python = ">=3.11" license = "Apache-2.0" authors = [{ name = "NVIDIA" }] -# Each owns a format the collector would otherwise parse by hand. Pinned: the -# documents they produce are published, so an upgrade is a deliberate change. +# Pin format libraries to keep published documents reproducible across builds. dependencies = [ "installer==1.0.1", # RECORD "license-expression==30.4.4", # SPDX expression grammar "packageurl-python==0.17.6", # purls "packaging==26.3", # core metadata, requirements "pyelftools==0.33", # ELF dynamic section (auditwheel uses it too) - # REUSE metadata of checked-in third-party files. Keep in step with the - # reuse-tool rev in .pre-commit-config.yaml, which lints that metadata. Reads - # files through python-magic, so `build` needs libmagic1 on the runner. + # Match reuse-tool in .pre-commit-config.yaml; build also needs libmagic1. "reuse==6.2.0", ] [project.optional-dependencies] -# Third-party conformance check for the documents this emits. dev = ["pytest", "spdx-tools==0.8.3"] [project.scripts] diff --git a/scripts/sbom/tests/conftest.py b/scripts/sbom/tests/conftest.py index 8b00ad66a5..8954abb097 100644 --- a/scripts/sbom/tests/conftest.py +++ b/scripts/sbom/tests/conftest.py @@ -16,7 +16,6 @@ def repo_root() -> Path | None: - """The source tree these tests sit in: the first ancestor with VERSION and CMakeLists.txt.""" for parent in Path(__file__).resolve().parents: if (parent / "VERSION").is_file() and (parent / "CMakeLists.txt").is_file(): return parent @@ -24,10 +23,9 @@ def repo_root() -> Path | None: def _located(usable, missing: str, within=lambda build_dir: build_dir) -> Path: - """`within` the build tree BUILD_DIR_ENV names, else the checkout's `build/`, if `usable`. + """Use the selected build tree or default build/. - A build the environment names is one the caller promised, so its absence - fails every test; without one, the tests skip, since nothing was promised. + Missing explicit inputs fail; missing defaults skip. """ if os.environ.get(BUILD_DIR_ENV): named = within(Path(os.environ[BUILD_DIR_ENV]).resolve()) @@ -62,20 +60,14 @@ def license_data(): @pytest.fixture(autouse=True) def _license_data_loaded(license_data): - """Load the reference texts before each test. - - Reloading matters: a `build()` call points the collector at its own build - tree, which for the fixtures holds only a handful of licenses. Without this - the next test would inherit that subset. - """ + """Restore the full corpus after tests that load a fixture subset.""" licensing.load_corpus(license_data) synth.set_license_data(license_data) @pytest.fixture(autouse=True) def _fresh_checkout_answers(): - """Each test builds its own checkouts and fakes its own host; a cached - answer is a previous test's.""" + """Clear cached checkout and host answers between synthetic workspaces.""" discovery._is_own_checkout.cache_clear() discovery._apt_releases.cache_clear() discovery._apt_origin.cache_clear() @@ -83,7 +75,6 @@ def _fresh_checkout_answers(): @pytest.fixture(scope="session") def checkout(): - """This repository, as the collector would be pointed at it.""" root = repo_root() if root is None: pytest.skip("not inside this repository's source tree") @@ -92,12 +83,7 @@ def checkout(): @pytest.fixture(scope="session") def live_build_dir(): - """This repository's own configured build tree, when there is one. - - The synthetic workspaces are written to a model of what CMake emits. This is - the anchor to what it actually emits, so a change to the codemodel's shape - fails here instead of passing everywhere. - """ + """Use real CMake output to validate the synthetic codemodel assumptions.""" return _located(_has_codemodel, "no CMake file API codemodel") @@ -111,5 +97,4 @@ def workspace(tmp_path): @pytest.fixture def built(workspace): - """The workspace after the collector has packaged evidence into the wheel.""" return synth.collect_and_bind(workspace) diff --git a/scripts/sbom/tests/synth.py b/scripts/sbom/tests/synth.py index 80c3655554..7fdd89279d 100644 --- a/scripts/sbom/tests/synth.py +++ b/scripts/sbom/tests/synth.py @@ -1,18 +1,10 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -"""A synthetic project, build tree and wheel for the SBOM collector's tests. - -The fixture has to be a build tree: a file API reply, fetched source trees, an -SDK tarball, a staged package directory. Generated rather than built, because the -gates need inputs a working build cannot produce -- a member from nowhere, a -component with no license text, a tampered wheel -- and because the suite must -run with no compiler, no network and no configured CMake. - -Keep every reader anchored to something real as well, or a fixture the collector -agrees with proves nothing: the codemodel reader against this repo's own build -tree, the ELF reader against a system library, the matcher against the fetched -SPDX corpus. +"""Generate synthetic build trees and wheels, including malformed gate inputs. + +Check reader assumptions separately against real CMake replies, ELF files, +and the SPDX corpus. """ from __future__ import annotations @@ -40,9 +32,7 @@ SDK_ARCHIVE = "deps/vendor/demosdk-1.2.0-linux-amd64.tar.gz" SDK_DEMO_BUILD_ID = "1111111111111111111111111111111111111111" -# ============================================================================== # ELF -# ============================================================================== _PT_LOAD = 1 _PT_DYNAMIC = 2 @@ -78,11 +68,7 @@ def elf_shared_object( padding: bytes = b"", build_id: str | None = None, ) -> bytes: - """A 64-bit little-endian ELF carrying a readable dynamic segment. - - With `build_id`, it also carries the note the linker writes -- which is what - survives a repair tool rewriting SONAME and RPATH. - """ + """Build a little-endian ELF64 dynamic segment with an optional GNU build-ID note.""" strings = [b"\0"] # index 0 is the empty string, per the ELF string-table format offsets: dict[str, int] = {} cursor = 1 @@ -149,9 +135,7 @@ def phdr(index: int, kind: int, offset: int, size: int) -> None: return bytes(image) -# ============================================================================== -# License texts, taken from the same corpus the collector matches against -# ============================================================================== +# License texts _LICENSE_DATA: Path | None = None @@ -176,7 +160,6 @@ def phdr(index: int, kind: int, offset: int, size: int) -> None: def set_license_data(path: Path) -> None: - """Point the fixtures at the SPDX json/ directory the build materialized.""" global _LICENSE_DATA _LICENSE_DATA = path @@ -204,9 +187,7 @@ def spdx_text(license_id: str) -> str: """ -# ============================================================================== # Workspace -# ============================================================================== _DEPS = { # key: (remote, license file name, SPDX id whose canonical text it ships) @@ -294,10 +275,7 @@ def __init__(self, root: Path) -> None: def git(root: Path, *args: str) -> str: - """Run git in `root` and return its output.""" - # Fixed identity *and* clock: a commit hashes its timestamp, the collector - # records the commit, and the wheel carries that record -- so a drifting - # clock makes two builds of the same project differ. + # Fix identity and timestamps for reproducible synthetic commits and wheel provenance. env = { **os.environ, "GIT_AUTHOR_NAME": "synth", @@ -322,10 +300,9 @@ def _commit_all(root: Path, message: str) -> None: def clone_of(root: Path, remote: str, tag: str, git_dir: Path | None = None) -> str: - """Make `root` a checkout of its own, as FetchContent's clone leaves it. + """Create a dependency checkout and return its commit ID. - `git_dir` puts the repository elsewhere behind a `.git` file, the way a - submodule's is. Returns the commit. + git_dir uses a separate repository behind a .git file, as in a submodule. """ root.mkdir(parents=True, exist_ok=True) separate = [] @@ -340,13 +317,7 @@ def clone_of(root: Path, remote: str, tag: str, git_dir: Path | None = None) -> def _git_checkout(root: Path, initialise: bool = True) -> None: - """Make the generated project a real checkout. - - The collector separates repository source from build output by what git - tracks, because an install prefix carries no marker saying it is output. - `build/` and `dist/` are ignored here for the same reason they are in the - real repository. - """ + """Create a real checkout to distinguish tracked sources from build output.""" write(root / ".gitignore", "build/\ndist/\n") if initialise: git(root, "init", "-q", "-b", "main") @@ -375,7 +346,6 @@ def reuse_toml(*annotations: str) -> str: def track(root: Path) -> None: - """Commit whatever a test added after the project was generated.""" _git_checkout(root, initialise=False) @@ -468,9 +438,7 @@ def _sdk_tarball(path: Path) -> dict[str, bytes]: "libextra.so": elf_shared_object("libextra.so", ("libssl.so.3", "libc.so.6")), "include/demo.h": b"#pragma once\n", } - # Fixed mtimes, on the members and on the gzip stream: "w:gz" stamps the - # current time into both, and two archives written a second apart are not - # the same input. + # Fix member and gzip timestamps for reproducible archive bytes. path.parent.mkdir(parents=True, exist_ok=True) with ( gzip.GzipFile(filename=path, mode="wb", mtime=0) as compressed, @@ -485,7 +453,6 @@ def _sdk_tarball(path: Path) -> dict[str, bytes]: def create(root: Path) -> Workspace: - """Generate the project, its build tree, and the staged package.""" workspace = Workspace(root) build = workspace.build deps = build / "_deps" @@ -508,13 +475,12 @@ def create(root: Path) -> Workspace: write(source / f"{key}.cpp", f"// {key}\n") declare(deps, key, repository=remote, tag="v1.2.3") - # vendorpy ships Python that the build stages into the wheel. write(deps / "vendorpy-src/pkg/mod.py", _VENDOR_MODULE) write(deps / "vendorpy-src/pkg/assets/hand.xml", _VENDOR_ASSET_SOURCE) sdk_members = _sdk_tarball(root / SDK_ARCHIVE) - # --- build outputs ------------------------------------------------------- + # Build outputs extension = "isaaccapture/_ext.cpython-312-x86_64-linux-gnu.so" extension_bytes = elf_shared_object( "_ext.cpython-312-x86_64-linux-gnu.so", ("libstdc++.so.6", "libc.so.6") @@ -524,13 +490,10 @@ def create(root: Path) -> Workspace: write(build / "lib/libgamma_renamed.so", gamma_bytes) write(build / "lib/libdelta.a", b"!\ndelta\n") write(build / "generated/_generated.py", _GENERATED) - # Empty, like the real PEP 561 marker: RECORD states size 0 for it, and the - # bytes explain nothing, so it also stands in for any zero-length member. + # Empty marker exercises RECORD size 0 and path-only attribution. write(build / "generated/stubs/isaaccapture/py.typed", b"") - # Staged tree: what the wheel is built from. - # What a package manager installs into the build tree: upstream headers - # carrying their own contributors' copyright lines. + # Model a package-manager header installed into the build tree. write( build / "vcpkg_installed/x64-linux/include/eigen/Core.h", b"// Copyright (C) 2008 Gael Guennebaud \n", @@ -562,11 +525,7 @@ def create(root: Path) -> Workspace: def _file_api_reply(root: Path, build: Path, extension: str) -> None: - """Write the codemodel a `cmake_file_api()` query leaves behind. - - The collector reads CMake's file API rather than scraping generator output, - so the fixture has to speak the same thing a real configure produces. - """ + """Write a synthetic CMake file API reply.""" targets = [ { "name": "ext", @@ -574,9 +533,7 @@ def _file_api_reply(root: Path, build: Path, extension: str) -> None: # The project's own source, compiled against beta's headers, with # alpha's source compiled straight in. "artifacts": [extension], - # The vcpkg port is listed as a source, not just an include: a - # package manager installs into the build tree, and the codemodel - # cites what it compiled. + # List a package-manager header as a source to exercise build-tree exclusion. "sources": [ "src/cpp/module.cpp", "build/_deps/alpha-src/alpha.cpp", @@ -632,7 +589,6 @@ def _file_api_reply(root: Path, build: Path, extension: str) -> None: "configurations": [{"name": CONFIG, "targets": index}], } write(reply / "codemodel-v2-fixture.json", json.dumps(codemodel, indent=1)) - # The collector enters through the index, as a file API client must. write( reply / "index-2026-01-01T00-00-00-0000.json", json.dumps( @@ -678,13 +634,11 @@ def _target_payload( def _target_entry(name: str) -> dict: - """A target's entry in the codemodel's configuration.""" return {"id": name, "name": name, "jsonFile": f"target-{name}-{CONFIG}.json"} @contextlib.contextmanager def _edit_json(path: Path): - """Read a reply file, let the caller change it, write it back.""" payload = json.loads(path.read_text(encoding="utf-8")) yield payload path.write_text(json.dumps(payload, indent=1), encoding="utf-8") @@ -775,13 +729,11 @@ def set_toolchains( def add_include(build: Path, target: str, directory: Path) -> None: - """Put one more directory on a fixture target's include path.""" with _edit_target(build, target) as payload: payload["compileGroups"][0]["includes"].append({"path": str(directory)}) def add_source(build: Path, target: str, path: Path) -> None: - """Compile one more source into a fixture target.""" with _edit_target(build, target) as payload: payload["sources"].append(_source(str(path))) @@ -851,19 +803,16 @@ def add_target( def set_directory(build: Path, target: str, directory: str) -> None: - """Put a fixture target in a subdirectory of the build tree.""" with _edit_target(build, target) as payload: payload["paths"] = {"build": directory, "source": directory} def set_type(build: Path, target: str, kind: str) -> None: - """Change a fixture target's CMake type.""" with _edit_target(build, target) as payload: payload["type"] = kind def set_generator(build: Path, name: str) -> None: - """Record the generator in the file API index, as a real configure does.""" with _edit_index(build) as index: index["cmake"] = {"generator": {"name": name, "multiConfig": False}} @@ -986,9 +935,7 @@ def ar_archive(members: dict[str, bytes]) -> bytes: return bytes(image) -# ============================================================================== # Wheel -# ============================================================================== METADATA = "\n".join( [ @@ -1001,8 +948,7 @@ def ar_archive(members: dict[str, bytes]) -> bytes: "License-Expression: Apache-2.0", "License-File: LICENSE.md", "Requires-Python: >=3.11", - # The transition wheel requires this one and this one requires it back; - # a cycle is the shape the real pair has. + # Model the cyclic dependency between the main and transition wheels. "Requires-Dist: isaacteleop>=0.4.0", "Requires-Dist: numpy>=1.23.0", "Requires-Dist: pyyaml>=6.0.3", @@ -1040,15 +986,12 @@ def wheel_payload(workspace: Workspace) -> dict[str, bytes]: workspace.root / "LICENSE.md" ).read_bytes() payload[f"{DIST_INFO}/top_level.txt"] = b"isaaccapture\nvendorpy\n" - # auditwheel runs before the collector and writes its own PEP 770 document - # for the libraries it vendored. Every fixture carries it so the suite works - # on the shape a released wheel actually has. + # Include auditwheel's PEP 770 document to test coexistence. payload[f"{DIST_INFO}/sboms/{AUDITWHEEL_SBOM}"] = AUDITWHEEL_CDX return payload def collect(workspace: Workspace, **options) -> dict: - """Run the collector over the workspace's wheel and read back what it wrote.""" out_dir = workspace.root / "sbom" manifest = build_module.build( workspace.root, workspace.build, workspace.wheel, out_dir, **options @@ -1138,8 +1081,7 @@ def write_wheel(path: Path, payload: dict[str, bytes]) -> Path: writer.writerow((name, _record_hash(payload[name]), str(len(payload[name])))) writer.writerow((record_name, "", "")) - # Fixed stamps: writestr would otherwise record wall-clock, and two wheels - # written either side of a second boundary are not the same input. + # Fix ZIP timestamps for reproducible wheel bytes. path.parent.mkdir(parents=True, exist_ok=True) with zipfile.ZipFile(path, "w", zipfile.ZIP_DEFLATED) as archive: for name in [*sorted(payload), record_name]: diff --git a/scripts/sbom/tests/test_discovery.py b/scripts/sbom/tests/test_discovery.py index 9068d6c41a..bcc8fa64ba 100644 --- a/scripts/sbom/tests/test_discovery.py +++ b/scripts/sbom/tests/test_discovery.py @@ -1,7 +1,7 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -"""What the build left behind is the only dependency declaration there is.""" +"""Build provenance discovery tests.""" from __future__ import annotations @@ -20,8 +20,6 @@ def test_source_trees_are_found_without_being_listed(workspace): components = discovery.discover_source_trees(workspace.build / "_deps") - # license-list-data is the SPDX reference data deps/third_party fetches. It is - # fetched the same way as everything else, so it is discovered the same way. assert set(components) == { "alpha", "beta", @@ -57,11 +55,7 @@ def test_identity_comes_off_the_remote_url(workspace): def test_an_archive_fetched_inside_a_checkout_is_not_that_checkout( workspace, monkeypatch, hook ): - """git answers for the repository enclosing `build/`; the declaration does not. - - A hook runs with GIT_DIR exported, which makes every directory look like - the root of that repository. - """ + """GIT_DIR can override -C, even for an archive nested under the project checkout.""" deps = workspace.build / "_deps" synth.write(deps / "sdl2-src/LICENSE.txt", synth.spdx_text("Zlib")) synth.declare(deps, "sdl2", urls=(SDL_RELEASE,), digest=SDL_HASH) @@ -511,8 +505,6 @@ def test_a_toolkit_licence_beside_a_library_is_not_a_sidecar(tmp_path): def test_a_build_tree_without_a_file_api_reply_says_so(workspace): - """The query lives in CMakeLists, so a missing reply means the tree was - configured by a CMake too old to honour it — say that, don't guess.""" import shutil shutil.rmtree(workspace.build / ".cmake") @@ -522,8 +514,7 @@ def test_a_build_tree_without_a_file_api_reply_says_so(workspace): def test_a_stale_codemodel_beside_the_current_one_is_not_read(workspace): - """Reply filenames are unspecified, so the index is the only thing that says - which codemodel belongs to this configure. A leftover one sorts later here.""" + """The stale codemodel sorts later; only the index selects the current one.""" reply = workspace.build / ".cmake" / "api" / "v1" / "reply" stale = json.loads((reply / "codemodel-v2-fixture.json").read_text()) stale["configurations"] = [{"name": synth.CONFIG, "targets": []}] @@ -535,15 +526,11 @@ def test_a_stale_codemodel_beside_the_current_one_is_not_read(workspace): def test_a_configuration_the_codemodel_does_not_describe_is_refused(workspace): - """The cache names the configuration; a reply that does not describe it is a - reply for another build, and attributing from it would credit the wrong one.""" with pytest.raises(discovery.FileApiError, match="Debug"): discovery.BuildGraph(workspace.build, "Debug") def test_the_graph_is_read_from_the_codemodel_not_from_generator_output(workspace): - """link.txt is a Makefile-generator artifact; Ninja writes none. Reading the - file API is what makes the collector work under either.""" assert not list(workspace.build.rglob("link.txt")) assert not (workspace.build / "compile_commands.json").exists() @@ -554,10 +541,7 @@ def test_the_graph_is_read_from_the_codemodel_not_from_generator_output(workspac def test_third_party_checked_in_here_is_still_third_party(workspace): - """Not everything third-party arrives through FetchContent. A file checked - into the repository that someone else wrote says so in its REUSE info, and - that is what separates it from the project's own code -- not the directory - it sits in.""" + """Copyright metadata, not directory placement, identifies vendored code.""" from isaaccapture_sbom import evidence as evidence_module space = evidence_module.discover(workspace.root, workspace.build) @@ -567,7 +551,6 @@ def test_third_party_checked_in_here_is_still_third_party(workspace): if component.kind == "vendored-source" } - # Named for what it is, not as though the holder were a product. assert [c.name for c in vendored.values()] == [ "Vendored source (Upstream Widgets Ltd)" ] @@ -576,12 +559,9 @@ def test_third_party_checked_in_here_is_still_third_party(workspace): ] component = next(iter(vendored.values())) assert component.license_concluded == "BSL-1.0" - # The text for the identifier the file declares comes from the SPDX list this - # build pinned -- never from this repository's own REUSE pool, whose copies - # license this project and carry its notices. + # Use upstream reference texts, not the repository's own license pool. assert [item.path for item in component.evidence] == ["BSL-1.0.txt"] assert all(item.origin == "spdx-reference" for item in component.evidence) - # The notice the component states about itself is recorded, not dropped. assert "Upstream Widgets Ltd" in component.copyright_text @@ -600,11 +580,7 @@ def test_a_vendored_header_is_credited_to_the_artifact_that_includes_it(workspac def test_the_codemodel_reader_works_on_a_codemodel_cmake_wrote( live_build_dir, checkout ): - """Reads this repository's own build tree, not a synthesized one. - - Deliberately makes no claim about which dependencies are present -- only - about the shape of what the reader gets back, so a bump cannot break it. - """ + """Check real codemodel structure without depending on the current dependency set.""" graph = discovery.BuildGraph(live_build_dir) assert graph.artifacts, "a configured build declares at least one artifact" @@ -618,19 +594,13 @@ def test_the_codemodel_reader_works_on_a_codemodel_cmake_wrote( for how in graph.contributions(artifact).values(): assert how <= {"compiled-source", "header-include"} - # Relative link-line paths resolve against the directory the link ran in; - # resolved against any other, an archive this build made matches no target. + # Relative link paths must resolve to this build's indexed outputs. named = {path for node in graph._nodes.values() for path in node.linked} assert named & set(graph.artifacts), "this project links static libraries it builds" def test_a_second_build_tree_is_not_mistaken_for_source(workspace, license_data): - """A developer's other build directory holds staged copies of the repo. - - Walking those as source makes a file's origin ambiguous, which surfaces as - an unexplained wheel member rather than a wrong one. A configured tree says - what it is with a CMakeCache.txt. - """ + """A second configured build contains staged source copies that must not enter the source index.""" from isaaccapture_sbom import evidence, licensing licensing.load_corpus(license_data) @@ -650,12 +620,7 @@ def test_a_second_build_tree_is_not_mistaken_for_source(workspace, license_data) def test_a_fetched_archive_records_where_it_came_from(workspace, license_data): - """Archives here are downloaded at configure time, not checked in. - - The fetch script writes the URL beside the tarball, so the origin is read - rather than guessed; without that file the document must say it does not - know. - """ + """The fetch script preserves archive origin in a .source sidecar.""" from isaaccapture_sbom import evidence, licensing licensing.load_corpus(license_data) @@ -673,15 +638,11 @@ def test_a_fetched_archive_records_where_it_came_from(workspace, license_data): after = known.components[synth.SDK_ARCHIVE] assert after.download_location.startswith("https://api.ngc.nvidia.com/") - # The registrable name, not the endpoint: SPDX supplier is who a recipient - # asks for source or terms, and a CDN hostname answers nobody. assert after.supplier == "Organization: nvidia" assert "fetched from https://" in after.source_info def test_a_build_tree_with_no_configuration_says_so(workspace): - """Guessing Release would read a directory that need not exist, and every - member it would have explained becomes an unexplained one instead.""" from isaaccapture_sbom import evidence cache = workspace.build / "CMakeCache.txt" @@ -699,7 +660,6 @@ def test_a_build_tree_with_no_configuration_says_so(workspace): def test_the_supplier_is_read_from_the_project_not_written_here(workspace): - """A fork or a rename must not leave the document naming the wrong party.""" assert discovery.project_supplier(workspace.root) == "Organization: NVIDIA" @@ -733,13 +693,7 @@ def test_a_project_naming_several_authors_stops_rather_than_choosing(workspace): def test_a_relative_build_dir_still_keeps_the_staged_tree_out( workspace, license_data, monkeypatch ): - """`--build-dir` is free-form, and the workflow passes a relative one. - - The staged package tree is what the wheel was built from, so indexing it - lets a member explain itself -- the one route around the gate every other - member has to pass. The exclusion compared an absolute skip path against - relative walked ones and silently matched nothing. - """ + """The workflow passes a relative build directory; staging must still be excluded.""" from isaaccapture_sbom import evidence, licensing licensing.load_corpus(license_data) @@ -755,13 +709,7 @@ def test_a_relative_build_dir_still_keeps_the_staged_tree_out( def test_a_dependency_installed_into_the_build_tree_is_not_vendored_source( workspace, license_data ): - """vcpkg installs its ports under the build directory. - - Those headers carry their upstream contributors' copyright lines. Read as - repository source they become one "vendored" component per person -- seventy - of them on a real build -- each then failing the gate for shipping no licence - text of its own. - """ + """Installed dependency headers must not become one vendored component per copyright holder.""" from isaaccapture_sbom import evidence, licensing licensing.load_corpus(license_data) @@ -775,13 +723,7 @@ def test_a_dependency_installed_into_the_build_tree_is_not_vendored_source( def test_vendored_material_asserts_no_package_identifier(workspace, license_data): - """A copyright holder is not a package. - - Checked-in third-party material is identified by the copyright holder its - REUSE info or header names, because that is the only identity it has. Minting - `pkg:generic/` from it named a package that does not exist, and a - reader resolving that locator finds nothing or finds something else. - """ + """Copyright ownership does not establish a resolvable package identity.""" from isaaccapture_sbom import licensing licensing.load_corpus(license_data) @@ -798,7 +740,6 @@ def test_vendored_material_asserts_no_package_identifier(workspace, license_data assert component.kind == "vendored-source" assert component.purl == "NOASSERTION" - # Who it came from is still stated; that much was observed. assert component.supplier == "Organization: Upstream Widgets Ltd" @@ -831,7 +772,6 @@ def _notices(components) -> dict[str, set[str]]: _LICENCE_TAG = "SPDX-" + "License-Identifier: MIT" _IGNORED_BAR = "// REUSE-IgnoreStart\n// Copyright 2020 Bar Ltd\n// REUSE-IgnoreEnd\n" -# Third-party headers, by what REUSE can read of them. _HEADERS = { "copyright-and-licence": f"// Copyright 2020 Bar Ltd\n// {_LICENCE_TAG}\n", "copyright-only": "// Copyright 2020 Bar Ltd\n", @@ -869,9 +809,7 @@ def _notices(components) -> dict[str, set[str]]: "files", [pytest.param(files, id=name) for name, files in _DECLARED.items()] ) def test_a_vendored_files_terms_are_what_reuse_declares(tmp_path, files): - """A header tag, a `.license` sidecar and a REUSE.toml annotation are one - declaration; each gives the identifier as written, `OR` and all, and the - reference text for every licence it names.""" + """All declaration forms preserve OR and supply every referenced text.""" for relative, text in files.items(): synth.write(tmp_path / relative, text) @@ -951,10 +889,6 @@ def test_a_vendored_files_terms_are_what_reuse_declares(tmp_path, files): def test_a_file_lending_nothing_is_named_not_covered_by_its_sibling( tmp_path, tag, reason ): - """A holder's licences are pooled for the component; a file whose own REUSE - info names a licence with no current SPDX reference text, states an - expression that does not parse, or names an id that is neither SPDX-listed - nor a LicenseRef- is still recorded, with why.""" (tmp_path / "src").mkdir() (tmp_path / "src/a.h").write_text( "// Copyright 2020 Foo Inc\n// SPDX-" + "License-Identifier: MIT\n" @@ -984,8 +918,7 @@ def test_a_listed_identifier_passes_the_per_file_gate(tmp_path, expression): def test_a_link_without_reuse_info_takes_its_targets(tmp_path): - """The REUSE lint skips symlinks and checks only the target, so a link with - no info of its own is read the same way; a link's own info still wins.""" + """A symlink's own REUSE metadata wins; without it, use the target's metadata.""" synth.write(tmp_path / "src/real/x.h", "// Copyright 2020 Foo Inc\n") _link(tmp_path, "src/link/x.h", "../real/x.h") synth.write( @@ -1125,7 +1058,6 @@ def test_a_header_counts_where_reuse_applies_it_beside_an_annotation( def test_declared_notices_naming_the_project_keep_the_file_first_party(tmp_path): - """The header's holder does not decide where a sidecar names the project.""" components = _vendored( tmp_path, { @@ -1199,7 +1131,6 @@ def test_license_files_and_the_pool_are_not_candidates(tmp_path): def test_a_build_input_reuse_does_not_cover_is_not_a_candidate(tmp_path): - """An include directory's licence text and sidecar are not sources.""" synth.write(tmp_path / "inc/x.h", "// Copyright 2020 Foo Inc\n") synth.write(tmp_path / "inc/x.h.license", f"SPDX-FileCopyrightText: {_FOO}\n") synth.write(tmp_path / "inc/LICENSE", "Copyright (c) 2020 Bar Ltd\n") @@ -1309,8 +1240,6 @@ def test_a_copyright_string_in_a_first_party_body_names_no_holder(tmp_path): ], ) def test_without_a_declared_holder_the_head_still_decides(tmp_path, files, expected): - """Where REUSE declares no copyright holder, a text file's first copyright - line names it, whether or not REUSE read that line.""" assert _notices(_vendored(tmp_path, files)) == expected diff --git a/scripts/sbom/tests/test_elf.py b/scripts/sbom/tests/test_elf.py index 25e22dcb3c..ad6941349f 100644 --- a/scripts/sbom/tests/test_elf.py +++ b/scripts/sbom/tests/test_elf.py @@ -1,7 +1,7 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -"""The ELF reader is how bundled libraries are told apart from external ones.""" +"""ELF metadata reader tests.""" from __future__ import annotations @@ -66,7 +66,6 @@ def test_verification_does_not_parse_binaries(built): assert not any(entry.is_elf for entry in plain.entries) assert any(entry.is_elf for entry in analyzed.entries) - # The digests verification relies on are identical either way. assert {e.name: e.sha256 for e in plain.entries} == { e.name: e.sha256 for e in analyzed.entries } diff --git a/scripts/sbom/tests/test_gates.py b/scripts/sbom/tests/test_gates.py index 00e36b6e87..60478fc789 100644 --- a/scripts/sbom/tests/test_gates.py +++ b/scripts/sbom/tests/test_gates.py @@ -30,7 +30,6 @@ def _rebuild(workspace, payload): def test_a_member_from_nowhere_blocks_the_build(workspace): - """A file that no build step and no fetched input explains is a coverage hole.""" payload = synth.wheel_payload(workspace) payload["isaaccapture/mystery/libmystery.so.7"] = synth.elf_shared_object( "libmystery.so.7" @@ -71,9 +70,7 @@ def test_an_empty_license_file_does_not_count_as_evidence(workspace): def test_a_library_the_host_cannot_resolve_is_not_claimed_as_the_host_s(workspace): - """Resolving the SONAME is the evidence for "the machine supplied this". - Without it the member is unexplained, and must be reported that way rather - than attributed to a library nobody found.""" + """Resolving the SONAME is required evidence for attribution to the build host.""" payload = synth.wheel_payload(workspace) payload["isaaccapture.libs/libunknown-a1b2c3.so"] = synth.elf_shared_object( "libunknown.so.1", ("libc.so.6",) @@ -96,9 +93,7 @@ def test_a_library_the_host_cannot_resolve_is_not_claimed_as_the_host_s(workspac def test_the_build_host_is_the_last_resort_not_a_directory_name(workspace): - """A library the build produced must be attributed to the build even when it - sits in a repair tool's directory: the host lookup runs only once every other - index has failed to explain the bytes.""" + """Build provenance takes precedence even inside a repair tool's directory.""" produced = (workspace.build / "lib/libgamma_renamed.so").read_bytes() payload = synth.wheel_payload(workspace) payload["isaaccapture.libs/libgamma_renamed-abc123.so"] = produced @@ -141,8 +136,7 @@ def test_a_vendored_library_is_attributed_to_the_package_that_supplied_it( "soname": soname, "resolved_path": "/usr/lib/libz.so.1.3", "package": "zlib1g", - # dpkg's maintainer, not the package name: `zlib1g` names the thing, - # not whoever supplied it. + # The package name identifies the software, not its supplier. "supplier": "Organization: Ubuntu Developers (ubuntu-devel@example.com)", "copyright": str(copyright_file), }, @@ -218,11 +212,7 @@ def test_our_own_document_going_missing_is_rejected(built): def test_one_wheel_described_by_several_builds_merges_to_one_record(tmp_path, built): - """A `py3-none-any` wheel is built by every matrix entry. - - Each entry's manifest names it, so a release set sees the filename once per - entry. Identical records are one wheel described more than once. - """ + """Matrix entries may produce identical universal wheels; merge their records.""" first = tmp_path / "a.json" second = tmp_path / "b.json" for path in (first, second): @@ -235,7 +225,6 @@ def test_one_wheel_described_by_several_builds_merges_to_one_record(tmp_path, bu def test_two_wheels_wearing_one_filename_are_refused(tmp_path, built): - """Same name, different bytes: the one thing a consumer cannot tell apart.""" first = tmp_path / "a.json" second = tmp_path / "b.json" first.write_text(json.dumps(built["manifest"]), encoding="utf-8") @@ -472,7 +461,6 @@ def test_a_wheel_the_manifest_does_not_list_fails_the_release_set( def test_missing_spdx_license_data_stops_the_build(workspace): - """The reference texts come from the build; without them, nothing publishes.""" shutil.rmtree(workspace.build / "_deps/license-list-data-src") with pytest.raises(licensing.CorpusError, match="SPDX license list data not found"): @@ -480,14 +468,12 @@ def test_missing_spdx_license_data_stops_the_build(workspace): def test_a_malformed_reuse_tag_does_not_reach_the_document(workspace): - """A hand-written tag can be unparseable; emitting it verbatim would fail - the published SBOM's own conformance check with no local signal.""" + """Reject malformed tags before emitting a document that fails SPDX conformance.""" payload = synth.wheel_payload(workspace) good = "# SPDX-" + "License-Identifier: mit or Apache-2.0\n" bad = "# SPDX-" + "License-Identifier: !!! not an expression\n" for name, text in (("tagged_ok.py", good), ("tagged_bad.py", bad)): payload[f"isaaccapture/{name}"] = text.encode() - # Authored sources, staged into the package like any other .py. (workspace.root / "src/python/isaaccapture" / name).write_text(text) (workspace.staged / "isaaccapture" / name).write_text(text) synth.track(workspace.root) @@ -500,13 +486,11 @@ def test_a_malformed_reuse_tag_does_not_reach_the_document(workspace): for item in spdx["files"] } - # Normalised through the SPDX grammar, not passed through as written. assert concluded["isaaccapture/tagged_ok.py"] == "MIT OR Apache-2.0" assert concluded["isaaccapture/tagged_bad.py"] == "NOASSERTION" def test_building_twice_on_the_same_wheel_is_refused(built): - """A second pass would duplicate members rather than replace them.""" with pytest.raises(build_module.BuildError, match="already carries"): build_module.build( built["workspace"].root, @@ -517,17 +501,13 @@ def test_building_twice_on_the_same_wheel_is_refused(built): def test_a_name_match_a_build_id_contradicts_is_refused(workspace, license_data): - """Two SDKs can ship a library under the same file name. - - The patched copy no longer hashes to either, so only the name is left to go - on -- and a differing build-id says it is the wrong one. Claiming it would - hand this binary the other component's licence. + """Patched libraries can lose hash identity while retaining a shared filename. + A contradictory build ID must prevent attribution by name. """ licensing.load_corpus(license_data) payload = synth.wheel_payload(workspace) - # Same basename as the member inside the SDK archive, different build-id, - # and patched so the content matches nothing. + # Match the SDK basename but contradict its build ID and content hash. payload["isaaccapture/other_sdk/libdemo.so"] = synth.elf_shared_object( "libdemo.so", ("libc.so.6", "libm.so.6"), @@ -545,8 +525,7 @@ def test_a_name_match_a_build_id_contradicts_is_refused(workspace, license_data) def test_an_unreadable_archive_says_so(workspace, license_data): - """Only archives this build fetched are indexed, so one that will not open - is a broken download -- not a stray file to walk past.""" + """Only fetched archives are indexed, so an unreadable one indicates a broken input.""" from isaaccapture_sbom import discovery licensing.load_corpus(license_data) @@ -564,11 +543,8 @@ def test_an_unreadable_archive_says_so(workspace, license_data): def test_bytes_more_than_one_component_holds_name_no_single_source( workspace, license_data ): - """A stock licence text is the same file in every project that ships one. - - Naming whichever was indexed first would file one project's redistribution - obligation under another's. Say the bytes are shared and claim no single - source instead. + """Identical license bytes can belong to several projects. + Do not assign ownership based on index order. """ licensing.load_corpus(license_data) @@ -592,20 +568,16 @@ def test_bytes_more_than_one_component_holds_name_no_single_source( assert "shared-content" in entry["comment"] assert "more than one component" in entry["comment"] - # Both candidates named, neither claimed as the source. assert "alpha" in entry["comment"] and "beta" in entry["comment"] def test_the_excluded_list_cannot_exempt_a_smuggled_file(built): - """The list lives in the document being checked. - - Treating it as files to skip lets whoever edits the document exempt anything - they add to the wheel from the only coverage check a consumer runs. + """The checked document controls its exclusion list; trusting it would let an editor + exempt arbitrary added files from coverage checks. """ with zipfile.ZipFile(built["wheel"]) as archive: payload = {item: archive.read(item) for item in archive.namelist()} - # write_wheel regenerates RECORD; keeping the old one would make a duplicate - # member, which is a different finding and now refused outright. + # write_wheel regenerates RECORD; remove it here to avoid testing duplicate members. payload.pop(f"{DIST_INFO}/RECORD", None) payload["isaaccapture/_backdoor.py"] = b"import os\n" @@ -626,10 +598,8 @@ def test_the_excluded_list_cannot_exempt_a_smuggled_file(built): def test_two_members_under_one_name_are_refused(built): - """Reading by name yields the last copy, so the first is hashed by nothing. - - Which one a consumer extracts is up to their unzip, so an archive carrying - both is not describable and must not be accepted. + """Name-based reads select one duplicate; extractors may select another. + No single hash can describe both members. """ with zipfile.ZipFile(built["wheel"]) as archive: names = [item for item in archive.namelist() if not item.endswith("/")] @@ -649,13 +619,8 @@ def test_two_members_under_one_name_are_refused(built): def test_one_discovery_serves_every_wheel_without_bleeding_between_them( workspace, tmp_path ): - """The build tree is read once for a run; a wheel's own findings are its own. - - `_from_build_host` adds the system libraries a wheel turned out to need. If - those land on the shared `Discovery`, the next wheel's evidence inherits - components it does not ship; if they land nowhere the evidence reaches, the - document names licence texts the evidence cannot account for and the wheel - fails its own verification. + """Host-library discoveries belong to each wheel's evidence. + Mutating shared discovery would leak components into subsequent wheels. """ from isaaccapture_sbom import evidence as evidence_module @@ -704,11 +669,8 @@ def components(wheel_name): def test_a_third_partys_licence_does_not_discharge_this_one(workspace, license_data): - """The distribution's own terms must be its own. - - A packaged third-party text with a matching basename satisfied the gate, so - a wheel that stopped shipping its own LICENSE still passed and the only copy - of its terms sat under `third-party/`, attributed to somebody else. + """A third-party license with the same basename cannot satisfy the requirement + to package the distribution's own terms. """ licensing.load_corpus(license_data) @@ -862,8 +824,7 @@ def test_merge_manifests_refuses_malformed_sidecars(tmp_path): def test_reuse_toml_declares_the_terms_of_an_untagged_vendored_file(workspace): - """A checked-in header that names no identifier, as Chromium's key table - does, under `src/vendor` so the extension that includes it ships it.""" + """Model an untagged vendored header such as Chromium's key table.""" synth.write( workspace.root / "src/vendor/keys/include/keys/table.h", synth.UNTAGGED_HEADER ) @@ -918,9 +879,9 @@ def test_reuse_toml_declares_the_terms_of_an_untagged_vendored_file(workspace): def test_each_vendored_file_needs_a_listed_licence_of_its_own( workspace, relative, text, key ): - """`src/vendor/upstream_helper.h` states BSL-1.0 for its holder; a file of - another holder, or a second file of theirs, stating nothing or only a - LicenseRef with no text, is not covered by it.""" + """A listed license on upstream_helper.h cannot cover other files that declare + no license or a LicenseRef without text. + """ synth.write(workspace.root / relative, text) synth.track(workspace.root) diff --git a/scripts/sbom/tests/test_licensing.py b/scripts/sbom/tests/test_licensing.py index e6d2769ea7..ca09c845fb 100644 --- a/scripts/sbom/tests/test_licensing.py +++ b/scripts/sbom/tests/test_licensing.py @@ -1,7 +1,7 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -"""Licenses are identified from the text a component ships, or not at all.""" +"""License text identification and declaration tests.""" from __future__ import annotations @@ -14,7 +14,6 @@ def test_corpus_loads(): - """The whole SPDX list is available, so no subset has to be maintained.""" version, entries = licensing.corpus() assert version.startswith("3.") @@ -23,8 +22,6 @@ def test_corpus_loads(): def test_deprecated_identifiers_are_not_offered(): - """Naming a component with a withdrawn identifier would be reporting a - license SPDX no longer recognises.""" _, entries = licensing.corpus() assert "GPL-2.0" not in entries, "superseded by GPL-2.0-only/-or-later" @@ -32,9 +29,10 @@ def test_deprecated_identifiers_are_not_offered(): def test_an_extended_license_is_not_mistaken_for_the_one_it_contains(): - """ECL-2.0 is Apache-2.0 plus a clause, and CC-BY-NC-4.0 is CC-BY-4.0 plus - one. Both clear containment against the shorter licence's own text, so the - file has to be resolved to the licence that explains all of it.""" + """ECL-2.0 contains Apache-2.0; CC-BY-NC-4.0 contains CC-BY-4.0. + + Containment alone cannot distinguish the extended license from its shorter base. + """ apache = [ license_id for license_id, _, _ in licensing.identify(synth.spdx_text("Apache-2.0")) @@ -49,7 +47,7 @@ def test_an_extended_license_is_not_mistaken_for_the_one_it_contains(): def test_the_extended_license_still_wins_when_it_is_the_real_one(): - """The resolution has to work in both directions, or it is just a bias.""" + """A longer match must not win solely because it is longer.""" matches = [ license_id for license_id, _, _ in licensing.identify(synth.spdx_text("ECL-2.0")) @@ -71,7 +69,6 @@ def test_identifies_a_text_wrapped_in_a_copyright_line(): def test_a_longer_license_subsumes_the_shorter_one_it_contains(): - """A BSD-3 file contains the BSD-2 text; only BSD-3 should be reported.""" matches = licensing.identify(synth.spdx_text("BSD-3-Clause")) assert [license_id for license_id, _, _ in matches] == ["BSD-3-Clause"] @@ -98,7 +95,6 @@ def _varied_prose(word_count: int) -> str: def test_a_license_quoted_in_an_appendix_is_not_the_document_s_license(): - """Large agreements reproduce whole OSS licenses; that is not their license.""" agreement = ( "PROPRIETARY AGREEMENT\n\n" + _varied_prose(6000) @@ -110,7 +106,6 @@ def test_a_license_quoted_in_an_appendix_is_not_the_document_s_license(): def test_a_real_license_survives_surrounding_packaging_boilerplate(): - """A Debian copyright file is mostly the license, wrapped in packaging prose.""" wrapped = ( "Format: https://www.debian.org/doc/packaging-manuals/copyright-format/1.0/\n" "Upstream-Name: thing\n\n" @@ -216,7 +211,6 @@ def _share(text: licensing.LicenseEvidence) -> str: def test_a_mostly_unmatched_grant_is_named_and_carried(): - """The rest of the text may be terms no id names.""" whole = _grant("demo", "LICENSE", synth.spdx_text("MIT")) padded = _grant("demo", "COPYING", synth.spdx_text("MIT") + _varied_prose(300)) zlib = _grant("demo", "COPYING.zlib", synth.spdx_text("Zlib") + _varied_prose(250)) @@ -486,10 +480,10 @@ def test_a_component_with_no_grant_declares_nothing(tmp_path): def test_a_tag_on_a_documentation_file_is_not_the_package_declaration(tmp_path): - """OpenXR's COPYING.adoc is prose about which licences the project uses, - tagged CC-BY-4.0 for the prose. Promoting it declared an Apache-2.0 SDK - under a documentation licence. The prose matches nothing, so it is carried - as the component's LicenseRef.""" + """OpenXR's COPYING.adoc is tagged CC-BY-4.0 for its prose. + + That tag does not license the SDK; unmatched prose travels as a LicenseRef. + """ (tmp_path / "LICENSE").write_text(synth.spdx_text("Apache-2.0"), encoding="utf-8") # REUSE-IgnoreStart (tmp_path / "COPYING").write_text( @@ -517,11 +511,9 @@ def test_a_tag_on_a_documentation_file_is_not_the_package_declaration(tmp_path): ], ) def test_two_identified_licenses_combine(files, concluded): - """Two identified licences must AND together, not collapse to NOASSERTION. + """Combined expression objects reject truth-value testing. - The combiner returns a parsed expression object, and asking such an object - for its truth value raises rather than answering -- so this path has to be - exercised with more than one licence. + Exercise the combiner with multiple licenses to catch accidental bool checks. """ evidence = [ _grant("dual", name, "\n\n".join(map(synth.spdx_text, ids))) @@ -529,7 +521,6 @@ def test_two_identified_licenses_combine(files, concluded): ] assert licensing.expression(evidence, "dual")[0] == concluded - # Nothing carried, so every grant is the terms the build evidence states. assert licensing.verbatim_terms(evidence) == "\n\n".join( item.text for item in evidence ) @@ -617,7 +608,6 @@ def test_a_license_suffix_is_a_sidecar_only_beside_what_it_annotates( def test_a_root_sidecar_is_not_a_fetched_tree_s_grant(tmp_path): - """Nor is it a README whose licence section the fallback could read.""" (tmp_path / "README.md").write_text("# Thing\n\nprose\n", encoding="utf-8") (tmp_path / "README.md.license").write_text( "SPDX-License-Identifier: MIT\n\n## License\nMIT\n", encoding="utf-8" @@ -651,7 +641,6 @@ def test_an_orphan_license_file_alone_is_a_fetched_tree_s_grant(tmp_path): def test_a_project_title_is_not_a_license_section(tmp_path): - """`# License List Data` heads a README; it does not grant anything.""" (tmp_path / "README.md").write_text( "# License List Data\n\nGenerated data.\n", encoding="utf-8" ) @@ -671,7 +660,6 @@ def test_a_real_license_section_is_still_found(tmp_path): def test_a_notice_another_already_covers_is_dropped(): - """One holder stated over several year ranges is one holder.""" folded = licensing.fold_notices( [ "Copyright (c) 2025 Acme Ltd.", @@ -699,7 +687,6 @@ def test_folding_does_not_merge_different_holders(): def test_a_template_is_not_a_notice(): - """A reference text names nobody; recording its placeholder would.""" assert licensing.read_notices("Copyright [yyyy] [name of copyright owner]") == [] assert licensing.read_notices("Copyright (c) ") == [] assert licensing.read_notices( @@ -708,12 +695,9 @@ def test_a_template_is_not_a_notice(): def test_a_debian_copyright_field_names_its_holder_either_way(license_data): - """Debian states a holder in a field, in two spellings, often with no year. + """Debian Copyright fields can omit years and contain prose or None. - `Copyright: Jane Doe` matches no notice pattern -- the colon breaks it -- and - it is the commoner of the two forms, so a system library vendored from such a - package was credited to nobody. The field also holds prose and the literal - `None`, which are not holders. + The colon in Copyright: prevents matching it as an ordinary notice. """ licensing.load_corpus(license_data) @@ -734,11 +718,7 @@ def test_a_debian_copyright_field_names_its_holder_either_way(license_data): def test_every_holder_in_a_text_is_read_not_just_the_first(license_data): - """Under MIT, BSD and Zlib the notice is the obligation, so all of it counts. - - Holders run on past a comma, sit on the next line with nothing joining them, - and follow a lead-in of any length. - """ + """Notices can span lines or follow long lead-ins; every holder must survive.""" licensing.load_corpus(license_data) assert licensing.read_notices( @@ -767,11 +747,8 @@ def test_every_holder_in_a_text_is_read_not_just_the_first(license_data): def test_a_choice_of_licences_is_not_published_as_both(license_data, tmp_path): - """`A OR B` is one licence at the reader's choice, not two obligations. - - Rebuilding a component's conclusion from the reference texts its identifiers - name loses the operator, and the roll-up that splits an expression into - identifiers loses it again. + """Preserve OR through matching and roll-up; rebuilding from identifiers would turn + a choice into simultaneous obligations. """ licensing.load_corpus(license_data) @@ -787,7 +764,6 @@ def test_a_choice_of_licences_is_not_published_as_both(license_data, tmp_path): assert component.license_concluded == "MIT OR GPL-2.0-or-later" assert component.license_declared == "MIT OR GPL-2.0-or-later" - # And it survives the roll-up whole, where an AND-only one decomposes. assert licensing.operands_of("MIT OR GPL-2.0-or-later") == [ "(MIT OR GPL-2.0-or-later)" ] @@ -798,10 +774,8 @@ def test_a_choice_of_licences_is_not_published_as_both(license_data, tmp_path): def test_an_exception_is_not_a_licence_needing_its_own_text(license_data): - """`X WITH ` must not be unsatisfiable. - - An exception's text ships only where its expression travels as a LicenseRef, - so demanding one for every exception blocks such a declaration for nothing. + """A WITH exception modifies a license; it is not a separate license operand + requiring its own packaged text in this gate. """ licensing.load_corpus(license_data) @@ -868,9 +842,7 @@ def test_the_debian_packaging_stanza_names_no_holder_of_the_library(): def test_a_field_paragraph_is_split_into_the_notices_it_states(): - """A notice wraps, so a line continues the one above unless it opens a - notice of its own -- and one line per holder is the common layout, which - folding needs to see one at a time.""" + """A wrapped line continues the preceding notice unless it starts a new one.""" text = ( "Format: https://www.debian.org/doc/packaging-manuals/copyright-format/1.0/\n\n" "Files: *\nCopyright: Copyright © 1980, 1982\n" diff --git a/scripts/sbom/tests/test_nested_terms.py b/scripts/sbom/tests/test_nested_terms.py index c860f262b8..0eb546f7e3 100644 --- a/scripts/sbom/tests/test_nested_terms.py +++ b/scripts/sbom/tests/test_nested_terms.py @@ -1,7 +1,7 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -"""Terms a component states for one part of itself ship only where that part did.""" +"""Scope nested license terms to the files contributed to the wheel.""" from __future__ import annotations @@ -61,7 +61,6 @@ def _kept(workspace, built=None) -> dict[str, dict | None]: for item in component["evidence"] if item["kind"] == "nested" } - # What was dropped is absent; name it so a test can say so. for root in sorted(_deps(workspace).glob("*-src")): for item in licensing.discover_in_tree(root.name, root, root.name): if item.kind == "nested": @@ -171,7 +170,6 @@ def test_without_a_record_per_object_include_directories_stand_in( assert kept["beta-src/include/COPYING"]["how"] == "include-dir" assert kept["beta-src/include/COPYING"]["path"] == "beta-src/include" - # Beneath the include directory too: any header under it may have been read. assert kept["beta-src/include/beta/fmt/LICENSE"]["how"] == "include-dir" assert kept["beta-src/docs/LICENSE"] is None @@ -372,7 +370,6 @@ def shown(key, path): ] assert shown("beta", deps / "beta-build/gen/other.h") == ["beta-src/gen/other.h"] assert shown("beta", deps / "beta-build/gen/blank.h") == ["beta-src/gen/blank.h"] - # A source-tree file is itself, whatever else holds its bytes. assert shown("beta", beta / "a/dup.h") == ["beta-src/a/dup.h"] assert shown("linked", deps / "linked-build/gen/config.h") == [ "linked-src/gen/config.h" diff --git a/scripts/sbom/tests/test_spdx_conformance.py b/scripts/sbom/tests/test_spdx_conformance.py index c03f888acc..ce0befbc81 100644 --- a/scripts/sbom/tests/test_spdx_conformance.py +++ b/scripts/sbom/tests/test_spdx_conformance.py @@ -1,7 +1,7 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -"""Third-party conformance check, so the document is valid SPDX and not just ours.""" +"""Validate generated documents with the third-party SPDX validator.""" from __future__ import annotations @@ -63,6 +63,5 @@ def test_an_identifier_newer_than_the_grammar_still_validates(workspace): item["component"]: item for item in report["components_with_unidentified_license"] } - # The id that caused the fallback is named, and its text ships. assert "rsync-linking-exception" in gaps["vendored:rsync-folk"]["reason"] assert "rsync-linking-exception.txt" in gaps["vendored:rsync-folk"]["evidence"] diff --git a/scripts/sbom/tests/test_static_archives.py b/scripts/sbom/tests/test_static_archives.py index 849e59d212..42735edd27 100644 --- a/scripts/sbom/tests/test_static_archives.py +++ b/scripts/sbom/tests/test_static_archives.py @@ -1,7 +1,7 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -"""What a link line names is a contribution, wherever it came from.""" +"""Static link inputs, linker resolution, and package ownership tests.""" from __future__ import annotations @@ -52,7 +52,6 @@ def value(item): def _row(collected: dict, key: str) -> dict: - """The licence report's row for one component.""" return next( item for item in collected["licenses"]["components"] if item["component"] == key ) @@ -225,8 +224,7 @@ def test_a_library_name_resolves_the_way_ld_resolves_it(workspace, tmp_path): "-lforced", ) - # -lboth takes the shared library, which DT_NEEDED records; after -Bstatic, - # -lforced takes the archive. + # -Bstatic selects an archive; the earlier shared library belongs to DT_NEEDED. assert {name: (item.kind, item.path) for name, item in inputs.items()} == { "-lforced": ("archive", searched / "libforced.a"), "-lstaticonly": ("archive", searched / "libstaticonly.a"), @@ -265,8 +263,7 @@ def test_every_spelling_of_a_search_resolves_alike(workspace, tmp_path): "-lspaced": joined / "libspaced.a", "-llinkerspaced": joined / "liblinkerspaced.a", "-lfromsep": separated / "libfromsep.a", - # Named outright, so judged by content: an archive without the suffix - # is still one, and a shared library is left to DT_NEEDED. + # Explicit names are classified by content, regardless of suffix. "-l:libarchive.a.1": joined / "libarchive.a.1", f"{tmp_path}/xl/libxl.a": tmp_path / "xl" / "libxl.a", } @@ -510,8 +507,7 @@ def test_a_script_naming_itself_says_so(workspace, tmp_path): @pytest.mark.parametrize("kind", ["EXECUTABLE", "MODULE_LIBRARY"]) def test_a_build_order_dependency_links_nothing_into_the_member(workspace, kind): - """A code generator ext waits for is not linked into it, so neither its line - nor a shared library reached only through it is.""" + """Build-order dependencies contribute no linked bytes, including their transitive libraries.""" synth.set_type(workspace.build, "unit_tests", kind) synth.link_line(workspace.build, "gamma", ("-l:libviagamma.a",)) synth.link_line( @@ -1034,7 +1030,6 @@ def test_a_tracked_archive_is_credited_to_the_holder_reuse_names( def test_a_tracked_archive_naming_no_other_holder_is_the_projects( workspace, monkeypatch ): - """Every tracked file has REUSE info; one naming nobody else is the project's.""" archive = _tracked_archive(workspace, monkeypatch) collected = synth.collect(workspace) @@ -1257,7 +1252,6 @@ def test_ownership_sees_a_file_registered_after_a_lookup(tmp_path): ownership.register_file(include / "b" / "b.h", "port-b", by_directory=False) ownership.register_file(include / "c.h", "fetched") - # Two listed owners under one directory credit neither; each alone does. assert ownership.owners_under(include) == {"fetched"} assert ownership.owners_under(include / "b") == {"port-b"} diff --git a/scripts/sbom/tests/test_wheel_sbom.py b/scripts/sbom/tests/test_wheel_sbom.py index 2ff9e6cdcb..a52658fd0f 100644 --- a/scripts/sbom/tests/test_wheel_sbom.py +++ b/scripts/sbom/tests/test_wheel_sbom.py @@ -1,7 +1,7 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -"""End to end: a repaired wheel goes in, a publishable wheel comes out.""" +"""End-to-end wheel evidence and verification tests.""" from __future__ import annotations @@ -41,7 +41,6 @@ def _wheel_package(spdx: dict) -> dict: def _release_failures(built) -> list[str]: - """What `check` says of the wheel with its build evidence and manifest.""" return validate_module.check( built["wheel"], build_evidence=built["evidence_path"], @@ -95,7 +94,6 @@ def test_every_member_records_where_it_came_from(built): def test_post_processed_files_are_recorded_as_derived_not_as_copies(built): - """patchelf and the MJCF rewrite change bytes; the claim has to change too.""" evidence = json.loads(built["evidence_path"].read_text(encoding="utf-8")) origins = {item["path"]: item for item in evidence["attributions"]} @@ -120,10 +118,8 @@ def test_a_dependency_reaching_no_wheel_file_is_recorded_by_the_build(built): components = built["build-evidence"]["components"] assert components["omega"]["in_this_wheel"] is False, "a test-only dependency" - # The SPDX reference data is fetched like any other dependency, so it is - # inventoried like any other dependency -- and ships nothing for the same reason. + # Fetched reference data belongs in build evidence, even though no wheel ships it. assert components["license-list-data"]["in_this_wheel"] is False - # What ships nothing is the build's to record, not the wheel's. assert "omega" not in json.dumps(built["licenses"]) assert "omega" not in built["md"] @@ -148,8 +144,7 @@ def test_licenses_are_packaged_and_declared(built): raw, metadata = wheelfile.read_metadata(built["wheel"], DIST_INFO) declared = set(metadata.get("license_files") or []) - # PEP 639 states these relative to .dist-info/licenses/, so each must - # resolve back to a member that is actually in the wheel. + # PEP 639 paths resolve relative to .dist-info/licenses/. prefix = f"{DIST_INFO}/licenses/" assert {name.removeprefix(prefix) for name in third_party} <= declared for value in declared: @@ -160,8 +155,7 @@ def test_licenses_are_packaged_and_declared(built): "the project's own license must survive the rewrite" ) - # No RFC 822 folding: a wrapped path is valid, and packaging unfolds it, but - # nothing else writes one and a continuation line defeats simpler readers. + # Avoid RFC 822 folding: simpler metadata readers may not unfold paths. assert not [ line for line in raw.decode().splitlines() if line.startswith((" ", "\t")) ] @@ -171,8 +165,7 @@ def test_an_unidentified_grant_travels_verbatim_in_the_document(built): spdx = _embedded(built["wheel"]) extracted = {item["licenseId"]: item for item in spdx["hasExtractedLicensingInfos"]} - # The id is minted from the component's key -- its path, here -- not from a - # basename two archives in different directories would share. + # Use the component path; archive basenames can collide. ref = next(item for item in extracted if "demosdk" in item) assert "deps" in ref and "vendor" in ref, ref assert "DEMO CORP" in extracted[ref]["extractedText"] @@ -219,8 +212,7 @@ def test_an_unmatched_grant_beside_a_matched_one_is_carried_and_reported( def test_a_component_shipping_only_a_reuse_pool_is_a_reported_gap(workspace): - """The pool passes the gate and concludes nothing, so it cannot fall - between the lists and leave the summary saying every licence is known.""" + """A REUSE pool passes the gate without concluding a license; report that gap.""" source = workspace.build / "_deps" / "alpha-src" (source / "LICENSES").mkdir() (source / "LICENSE").rename(source / "LICENSES" / "MIT.txt") @@ -242,7 +234,6 @@ def test_a_component_shipping_only_a_reuse_pool_is_a_reported_gap(workspace): assert entry["reason"].startswith("only REUSE pool texts ship (MIT)") [record] = built["manifest"]["wheels"] assert record["components_with_unresolved_license"] == [entry] - # The wheel's own conclusion cannot AND in terms nobody concluded. assert _wheel_package(built["spdx"])["licenseConcluded"] == "NOASSERTION" assert _release_failures(built) == [] @@ -407,10 +398,9 @@ def test_build_is_reproducible_under_source_date_epoch(tmp_path, monkeypatch): def test_source_date_epoch_normalizes_timestamps_the_input_wheel_carried( workspace, monkeypatch ): - """A zip records a local timestamp per member, so a wheel carries its - builder's clock. Under SOURCE_DATE_EPOCH the rewrite has to normalize the - members it copies through as well, or it is reproducible only for an input - wheel that already was.""" + """Normalize copied members too; otherwise reproducibility depends on the input wheel + already having reproducible timestamps. + """ import zipfile as zf stamped = synth.wheel_payload(workspace) @@ -451,12 +441,7 @@ def shared_text(workspace, license_data): def test_components_sharing_a_licence_text_share_one_entry(shared_text): - """A LicenseRef identifies a text, so one text is one entry. - - Two components can carry byte-identical terms -- two builds of one SDK, a - vendor licence applied across products -- and minting an id each duplicates - the whole text in the document. - """ + """Identical terms shared by several components need only one extracted text entry.""" spdx = _embedded(shared_text["wheel"]) bodies = [item["extractedText"] for item in spdx["hasExtractedLicensingInfos"]] @@ -497,14 +482,10 @@ def test_a_shared_id_is_held_to_the_text_it_names(shared_text): def test_a_requirement_this_run_also_built_is_not_an_unknown( workspace, license_data, tmp_path ): - """A transition wheel requires the wheel beside it; that is not an unknown. - - Left as an ordinary requirement it reads as something an installer resolves, - with version, supplier and licence unasserted -- for a distribution this same - build produced. Stated from the sibling's own metadata rather than by a - digest-bound reference to its document: the two wheels require each other, so - only one direction could ever carry such a reference, and a `py3-none-any` - wheel pinning one matrix entry's sibling stops being universal. + """Use sibling metadata for dependencies built in the same run. + + Digest-bound references cannot represent cycles, and a universal wheel must not + pin a platform-specific sibling from one matrix entry. """ from isaaccapture_sbom import cli @@ -562,7 +543,6 @@ def sibling_row(document, name): ) return spdx, row - # Both directions state the same kind of thing, and neither pins a document. forward, isaaccapture = sibling_row( "isaacteleop-0.4.0-py3-none-any.spdx.json", "isaaccapture" ) @@ -580,12 +560,8 @@ def sibling_row(document, name): def test_a_wheel_shipping_no_source_still_names_its_licensor(workspace, license_data): - """The declared licence's reference text names nobody. - - A metadata-only wheel ships no file carrying a notice, so without this the - SPDX package states NOASSERTION and the notices file says nothing about the - distribution -- a recipient holding only the wheel cannot tell who licensed - it to them, for a wheel that declares one. + """A metadata-only wheel has no source notices, and a reference license names no holder. + Use distribution metadata to identify its licensor. """ thin = workspace.wheel.parent / "thin-0.4.0-py3-none-any.whl" synth.write_wheel( @@ -625,7 +601,6 @@ def test_a_wheel_shipping_no_source_still_names_its_licensor(workspace, license_ assert "Example Org" in notices # The SPDX spelling of a supplier is not prose. assert "distributed by Organization:" not in notices - # The only text it ships names no holder, which is why the above matters. assert "thin-0.4.0.dist-info/licenses/LICENSES/Apache-2.0.txt" in members diff --git a/src/core/python/CMakeLists.txt b/src/core/python/CMakeLists.txt index d60ff4ea9e..dea5a3baff 100644 --- a/src/core/python/CMakeLists.txt +++ b/src/core/python/CMakeLists.txt @@ -12,8 +12,7 @@ if(NOT EXISTS "${ROBOTIC_GROUNDING_STAGED_DIR}/__init__.py") endif() message(STATUS "[grounding] bundling robotic_grounding/ from ${ROBOTIC_GROUNDING_STAGED_DIR}") -# project.license-files is stated once, in the root pyproject.toml, and both -# wheels ship that list: this one through pyproject.toml.in and the staging below. +# Share the root pyproject.toml's license-file list across both wheel builds. set(_root_pyproject "${CMAKE_SOURCE_DIR}/pyproject.toml") set(_license_files_script "${CMAKE_CURRENT_SOURCE_DIR}/license_files.py") set_property(DIRECTORY APPEND PROPERTY CMAKE_CONFIGURE_DEPENDS @@ -72,8 +71,7 @@ add_custom_target(python_package ALL COMMAND ${CMAKE_COMMAND} -E copy "${CMAKE_CURRENT_SOURCE_DIR}/MANIFEST.in" "${CMAKE_BINARY_DIR}/python_package/$/" COMMAND ${CMAKE_COMMAND} -E copy "${CMAKE_CURRENT_SOURCE_DIR}/setup.py" "${CMAKE_BINARY_DIR}/python_package/$/" COMMAND ${CMAKE_COMMAND} -E copy "${CMAKE_SOURCE_DIR}/README.md" "${CMAKE_BINARY_DIR}/python_package/$/README.md" - # Without these the wheel redistributes the project under a license it does - # not carry. A text dropped from the list must not linger from an older build. + # Remove stale license texts dropped from the list since the last build. COMMAND ${CMAKE_COMMAND} -E rm -rf "${CMAKE_BINARY_DIR}/python_package/$/LICENSES" ${_license_staging_commands} COMMAND ${CMAKE_COMMAND} -E copy "${CMAKE_CURRENT_SOURCE_DIR}/requirements.txt" "${CMAKE_BINARY_DIR}/python_package/$/" @@ -103,10 +101,7 @@ endif() set(STUBGEN_SCRIPT "${CMAKE_CURRENT_SOURCE_DIR}/generate_stubs.py") set(STUBGEN_PYPROJECT "${CMAKE_CURRENT_SOURCE_DIR}/stubgen_pyproject.toml") set(STUBGEN_PACKAGE_DIR "${CMAKE_BINARY_DIR}/python_package/$") -# Stubs are generated here and copied into the staging tree below. Writing them -# straight into staging would leave them with no build-tree origin, and the file -# a wheel ships must always be traceable to one -- staging is a copy destination, -# never the only place a file exists. +# Generate outside staging so the SBOM can trace each stub to a build-tree origin. set(STUBGEN_OUTPUT_DIR "${CMAKE_BINARY_DIR}/generated/stubs/$") set(PY_TYPED_MARKER "${STUBGEN_PACKAGE_DIR}/isaaccapture/py.typed") @@ -158,7 +153,7 @@ add_custom_command( # Create py.typed marker file (PEP 561) - also serves as build stamp COMMAND ${CMAKE_COMMAND} -E touch "${STUBGEN_OUTPUT_DIR}/isaaccapture/py.typed" - # Stage what was generated. Last, so PY_TYPED_MARKER stamps a complete copy. + # Copy last so PY_TYPED_MARKER stamps a complete staging tree. COMMAND ${CMAKE_COMMAND} -E copy_directory "${STUBGEN_OUTPUT_DIR}" "${STUBGEN_PACKAGE_DIR}" DEPENDS python_package "${STUBGEN_SCRIPT}" "${STUBGEN_PYPROJECT}" $ diff --git a/src/core/python/generate_stubs.py b/src/core/python/generate_stubs.py index 98d1ab250e..80f0f25384 100644 --- a/src/core/python/generate_stubs.py +++ b/src/core/python/generate_stubs.py @@ -28,18 +28,11 @@ def _err(message: str) -> None: def generate_stub(module_name: str, import_dir: Path, output_dir: Path) -> bool: - """Generate stub file for a single pybind11 module. + """Generate stubs for a fully qualified module imported from the staged package. - Args: - module_name: Fully qualified module name (e.g., "isaaccapture.deviceio._deviceio") - import_dir: Staged package tree to import the module from. - output_dir: Where to write the .pyi. Kept out of the staged tree so the - stub has a build-tree origin like every other generated file. - - Returns: - True if successful, False otherwise. + Keep output_dir outside staging to preserve a traceable build-tree origin. + Return whether generation succeeded. """ - # Import from the staged tree; write somewhere else entirely. sys.path.insert(0, str(import_dir)) try: diff --git a/src/core/python/license_files.py b/src/core/python/license_files.py index e0e19fa335..8a0b9f43a2 100644 --- a/src/core/python/license_files.py +++ b/src/core/python/license_files.py @@ -2,7 +2,7 @@ # SPDX-License-Identifier: Apache-2.0 """Print the root pyproject.toml's project.license-files for CMake. -The first line is the TOML array, verbatim, for pyproject.toml.in; each line +The first line is a JSON/TOML array for pyproject.toml.in; each line after it is one file those patterns match, relative to the repository root. """ diff --git a/src/core/python/pyproject.toml.in b/src/core/python/pyproject.toml.in index b3552833fc..e9ced4db91 100644 --- a/src/core/python/pyproject.toml.in +++ b/src/core/python/pyproject.toml.in @@ -9,9 +9,7 @@ # See: numpy.org/devdocs/dev/depending_on_numpy.html [build-system] -# setuptools>=77 for PEP 639: the SPDX `license` string and `license-files` -# below are what put LICENSE.md and LICENSES/ under .dist-info/licenses/ and -# name them in METADATA. The SBOM collector appends third-party texts there. +# setuptools>=77 implements PEP 639 license metadata and file placement. requires = ["setuptools>=77", "wheel", "numpy>=2.0"] build-backend = "setuptools.build_meta" diff --git a/tests/python/packaging/test_wheel_contents.py b/tests/python/packaging/test_wheel_contents.py index b1f2a9b526..0d0b93a12a 100644 --- a/tests/python/packaging/test_wheel_contents.py +++ b/tests/python/packaging/test_wheel_contents.py @@ -138,7 +138,6 @@ def test_capture_license_ships_in_distribution_metadata() -> None: def test_wheel_ships_exactly_the_root_license_files() -> None: - """Both wheel builds take their own license texts from the root pyproject.toml.""" root = repo_root() with (root / "pyproject.toml").open("rb") as handle: patterns = tomllib.load(handle)["project"]["license-files"]