diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 1bf76e5e..446aefc5 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -4,7 +4,6 @@ .github/* @simonbeaudoin0935 docker/* @simonbeaudoin0935 -scripts/* @vishwasudupa @bjordiscollaku @simonbeaudoin0935 rootfs/* @bjordiscollaku kernel/* @bjordiscollaku -bootloader/* @bjordiscollaku \ No newline at end of file +bootloader/* @bjordiscollaku diff --git a/.github/actions/abi_checker/action.yml b/.github/actions/abi_checker/action.yml deleted file mode 100644 index e8054265..00000000 --- a/.github/actions/abi_checker/action.yml +++ /dev/null @@ -1,95 +0,0 @@ -name: Check ABI Against Previous Package Version -description: | - This Github Actions checks the ABI of a newly built package against the latest - [or possibly a fixed] version of that same package stored a repository. - -inputs: - - apt-repository: - description: The apt repository to use for downloading the previous version of the package. For example "deb [arch=arm64 trusted=yes] https://qualcomm-linux.github.io/pkg-oss-staging-repo noble/stable main" - required: true - -runs: - using: "composite" - - steps: - - name : List All The Versions Of The Built Packages Contained In The Staging PPA - if: false # Disable for now - shell: bash - env: - INPUTS_APT_REPOSITORY: ${{ inputs.apt-repository }} - BUILT_PACKAGE_NAME: ${{ env.BUILT_PACKAGE_NAME }} - run: | - set +e - ./qcom-build-utils/scripts/ppa_interface.py \ - --operation list-versions \ - --apt-config "${INPUTS_APT_REPOSITORY}" \ - --package-name "${BUILT_PACKAGE_NAME}" - - RET=$? - set -e - - - name: ABI Check - shell: bash - env: - INPUTS_APT_REPOSITORY: ${{inputs.apt-repository}} - run: | - set +e - - ./qcom-build-utils/scripts/deb_abi_checker.py \ - --new-package-dir ./build-area \ - --apt-server-config "${INPUTS_APT_REPOSITORY}" \ - --result-file ./results.txt - - RET=$? - set -e - - echo "ABI check returned $RET" - - # (0): RETURN_ABI_NO_DIFF - # Bit 0 (1): RETURN_ABI_COMPATIBLE_DIFF - # Bit 1 (2): RETURN_ABI_INCOMPATIBLE_DIFF - # Bit 2 (4): RETURN_ABI_STRIPPED_PACKAGE - # Bit 3 (8): RETURN_PPA_PACKAGE_NOT_FOUND - # Bit 4 (16): RETURN_PPA_ERROR - - if (( RET == 0 )); then - echo "✅ ABI check returned NO_DIFF" - fi - - if (( RET & 1 )); then - echo "⚠️ ABI check returned COMPATIBLE DIFF" - fi - - if (( RET & 2 )); then - echo "⚠️ ABI check returned INCOMPATIBLE DIFF" - fi - - if (( RET & 4 )); then - echo "❌ ABI check returned STRIPPED PACKAGE" - exit 1 - fi - - if (( RET & 8 )); then - echo "⚠️ ABI check failed because the PPA did not contained an old version for the package." - echo "Assumption is that this is the first time the package was build." - fi - - if (( RET & 16 )); then - echo "❌ ABI check failed because there was an error on the PPA" - exit 1 - fi - - - name: Package Version Increment Check - shell: bash - run: | - echo "Run package version check here with ret value" - echo "Content of result file :" - cat ./results.txt - - if grep -qE '^\s*-\s*Version:\s*.*FAIL' ./results.txt; then - echo "❌ Test failed: At least one FAIL found in - Version: line" - exit 1 - else - echo "✅ Test passed: All versions are PASS" - fi diff --git a/.github/actions/build_package/action.yml b/.github/actions/build_package/action.yml deleted file mode 100644 index 5812ad17..00000000 --- a/.github/actions/build_package/action.yml +++ /dev/null @@ -1,305 +0,0 @@ -name: Build Debian Package -description: | - This Github Actions builds the package. - Supports two build modes, selected automatically or explicitly: - - Source build: Uses git-buildpackage (gbp) to build from source. - - Prebuilt binary: Downloads a binary tarball from Artifactory (defined in - upstream.conf) and packages it directly with sbuild, bypassing gbp. - Auto-detection: if prebuilt is left empty (default), the action checks whether - upstream.conf exists in the package directory. If found, prebuilt mode is used; - otherwise source build mode is used. - -inputs: - suite: - description: The distribution codename or Debian suite to build for. Ex noble, questing, trixie, etc - required: true - - pkg-dir: - description: The directory where the debian package source is - required: true - - build-dir: - description: The directory where the package is built - required: true - - run-lintian: - description: Run lintian or not during the build - default: false - - prebuilt: - description: | - Controls the build mode: - "true" — Force prebuilt binary mode: reads upstream.conf, downloads tarball, runs sbuild. - "false" — Force source build mode: runs gbp buildpackage. - "" — Auto-detect (default): uses prebuilt mode if upstream.conf exists in pkg-dir, - otherwise uses source build mode. - upstream.conf must export: ARTIFACTORY, TAG, DISTRO, PACKAGE_NAME. - Optionally export PACKAGE_SHA256 for checksum verification. - default: "" - -runs: - using: "composite" - - steps: - - # Normalize the name of the architecture - # Build Architecture: The architecture of the machine performing the build (arm64 when native or amd64 when cross compiling). - # This depends on the runner executing the build - - name: Set Builder Arch variable - shell: bash - run: | - if [ "${{ runner.arch }}" = "X64" ]; then - echo "BUILD_ARCH=amd64" >> $GITHUB_ENV - elif [ "${{ runner.arch }}" = "ARM64" ]; then - echo "BUILD_ARCH=arm64" >> $GITHUB_ENV - else - echo "Unsupported architecture: ${{ runner.arch }}" - exit 1 - fi - - - name: Detect Package Type - shell: bash - env: - INPUTS_PREBUILT: ${{inputs.prebuilt}} - INPUTS_PKG_DIR: ${{inputs.pkg-dir}} - run: | - if [[ "${INPUTS_PREBUILT}" == "true" ]]; then - echo "PREBUILT_MODE=true" >> $GITHUB_ENV - echo "ℹ️ Build mode: prebuilt (explicitly set)" - elif [[ "${INPUTS_PREBUILT}" == "false" ]]; then - echo "PREBUILT_MODE=false" >> $GITHUB_ENV - echo "ℹ️ Build mode: source (explicitly set)" - else - # Auto-detect based on presence of upstream.conf in the package directory - if [[ -f "${INPUTS_PKG_DIR}/upstream.conf" ]]; then - echo "PREBUILT_MODE=true" >> $GITHUB_ENV - echo "ℹ️ Build mode: prebuilt (auto-detected: upstream.conf found in ${INPUTS_PKG_DIR})" - else - echo "PREBUILT_MODE=false" >> $GITHUB_ENV - echo "ℹ️ Build mode: source (auto-detected: no upstream.conf in ${INPUTS_PKG_DIR})" - fi - fi - - - name: Download Prebuilt Binary Archive - if: ${{ env.PREBUILT_MODE == 'true' }} - shell: bash - env: - INPUTS_PKG_DIR: ${{inputs.pkg-dir}} - run: | - UPSTREAM_CONF="${INPUTS_PKG_DIR}/upstream.conf" - - if [[ ! -f "$UPSTREAM_CONF" ]]; then - echo "❌ upstream.conf not found at $UPSTREAM_CONF" - echo " This file is required when prebuilt=true." - echo " It must export: ARTIFACTORY, TAG, DISTRO, PACKAGE_NAME" - exit 1 - fi - - echo "ℹ️ Sourcing $UPSTREAM_CONF" - source "$UPSTREAM_CONF" - - # Validate all required variables are set after sourcing - for var in ARTIFACTORY TAG DISTRO PACKAGE_NAME; do - if [[ -z "${!var}" ]]; then - echo "❌ Required variable '$var' is not set in upstream.conf" - exit 1 - fi - done - - DOWNLOAD_URL="${ARTIFACTORY}/${TAG}/${DISTRO}/${PACKAGE_NAME}" - - echo "ℹ️ Downloading prebuilt binary archive:" - echo " URL : $DOWNLOAD_URL" - echo " Tag : $TAG" - echo " File : $PACKAGE_NAME" - - curl --fail --show-error --location \ - --output "${INPUTS_PKG_DIR}/$PACKAGE_NAME" \ - "$DOWNLOAD_URL" - - if [[ -n "$PACKAGE_SHA256" ]]; then - echo "ℹ️ Verifying SHA256 checksum..." - echo "$PACKAGE_SHA256 ${INPUTS_PKG_DIR}/$PACKAGE_NAME" | sha256sum --check - echo "✅ Checksum verified" - else - echo "⚠️ No PACKAGE_SHA256 defined in upstream.conf — skipping checksum verification" - fi - - echo "ℹ️ Extracting $PACKAGE_NAME into ${INPUTS_PKG_DIR}/" - tar -xf "${INPUTS_PKG_DIR}/$PACKAGE_NAME" -C "${INPUTS_PKG_DIR}" - rm -f "${INPUTS_PKG_DIR}/$PACKAGE_NAME" - echo "✅ Prebuilt archive extracted successfully" - - - name: Prepare Workspace Structure For The Build - shell: bash - env: - INPUTS_BUILD_DIR: ${{inputs.build-dir}} - run: | - echo "Listing the content of the workspace :"; tree - - mkdir -p ${INPUTS_BUILD_DIR} - - if grep -q 'quilt' ./package-repo/debian/source/format; then - echo "Source format is quilt" - elif grep -q 'native' ./package-repo/debian/source/format; then - echo "Source format is native" - else - echo "Source format is unknown or unsupported" - exit 1 - fi - - - name : Build Package - shell: bash - env: - INPUTS_PKG_DIR: ${{inputs.pkg-dir}} - INPUTS_RUN_LINTIAN: ${{inputs.run-lintian}} - INPUTS_BUILD_DIR: ${{inputs.build-dir}} - INPUTS_SUITE: ${{inputs.suite}} - run: | - cd ${INPUTS_PKG_DIR} - - # GitHub Actions container jobs use HOME=/github/home by default, but the - # builder images bake the sbuild config and unshare tarballs under /root. - export HOME=/root - - if [[ "${INPUTS_RUN_LINTIAN}" == "true" ]]; then - lintian_flag="--run-lintian" - else - lintian_flag="--no-run-lintian" - fi - - # Resolve the build output directory to an absolute path now, before any - # subsequent cd operations might change the working directory. - BUILD_DIR_ABS=$(realpath "../${INPUTS_BUILD_DIR}") - - set +e - - if [[ "$PREBUILT_MODE" == "true" ]]; then - if grep -q 'quilt' ./debian/source/format; then - # ℹ️ Prebuilt mode + 3.0 (quilt) source format: - # sbuild cannot build a quilt package without a .dsc and an upstream - # orig tarball. Replicate the same logic used by docker_deb_build.py - # when --skip-gbp is passed (make_source_pkg_cmd): - # 1. Create an upstream orig tarball from the source tree (excluding - # debian/ and .git/). - # 2. Run dpkg-source -b to produce the .dsc + debian tarball. - # 3. Pass the resulting .dsc to sbuild for the binary build. - PKG=$(dpkg-parsechangelog -l ./debian/changelog -S Source) - VER=$(dpkg-parsechangelog -l ./debian/changelog -S Version) - # Strip the debian revision to get the upstream version (e.g. '1.855.2-1' -> '1.855.2'). - UPSTREAM_VER=$(echo "$VER" | sed 's/-[^-]*$//') - SRC_ABS=$(realpath .) - WORK_DIR=$(dirname "$SRC_ABS") - - echo "[source-pkg] Package: $PKG Version: $VER Upstream: $UPSTREAM_VER" - echo "[source-pkg] Creating orig tarball: ${WORK_DIR}/${PKG}_${UPSTREAM_VER}.orig.tar.gz (may take a while for large trees)" - tar czf "${WORK_DIR}/${PKG}_${UPSTREAM_VER}.orig.tar.gz" \ - --exclude=./debian --exclude=./.git \ - -C "$SRC_ABS" . - - echo "[source-pkg] Running dpkg-source -b ..." - cd "$WORK_DIR" - dpkg-source -b "$SRC_ABS" - - DSC_FILE=$(ls "${WORK_DIR}/${PKG}_${VER}.dsc" 2>/dev/null | head -1) - [ -n "$DSC_FILE" ] || { echo "ERROR: .dsc not found after dpkg-source -b"; exit 1; } - echo "[source-pkg] Source package ready: $DSC_FILE" - - # Host Architecture: The architecture for which the binaries are being built (invariably arm64). - sbuild --no-clean-source \ - --host=arm64 \ - --build=${BUILD_ARCH} \ - --arch-all \ - --dist=${INPUTS_SUITE} \ - $lintian_flag \ - --build-dir "$BUILD_DIR_ABS" \ - --build-dep-resolver=apt \ - "$DSC_FILE" - else - # ℹ️ Prebuilt mode + native source format: invoke sbuild directly — - # no gbp wrapper or orig tarball needed. - # Host Architecture: The architecture for which the binaries are being built (invariably arm64). - sbuild --no-clean-source \ - --host=arm64 \ - --build=${BUILD_ARCH} \ - --arch-all \ - --dist=${INPUTS_SUITE} \ - $lintian_flag \ - --build-dir "$BUILD_DIR_ABS" \ - --build-dep-resolver=apt - fi - - else - # ℹ️ Source build mode. - # For packages with debian/watch (prebuilt binaries fetched from - # Artifactory), we mirror the debusine-action two-step approach: - # 1. Run uscan explicitly to fetch the upstream orig tarball. - # (dpkg-source --before-build calls uscan but does not reliably - # place the tarball where dpkg-source -b expects it in all envs.) - # 2. Run gbp buildpackage -S to produce the .dsc, with - # --git-no-create-orig so gbp does not overwrite the fetched tarball. - # 3. Pass the .dsc to sbuild for the binary build. - # For packages without debian/watch, gbp creates the orig from git as usual. - # Host Architecture: The architecture for which the binaries are being built (invariably arm64). - SRC_ABS=$(realpath .) - WORK_DIR=$(dirname "$SRC_ABS") - PKG=$(dpkg-parsechangelog -l ./debian/changelog -S Source) - VER=$(dpkg-parsechangelog -l ./debian/changelog -S Version) - - git config --global --add safe.directory "$SRC_ABS" - - # Only use the uscan fetch path for prebuilt binary packages that pull - # their upstream tarball from Artifactory. Packages with a watch file - # pointing to a git forge have their orig tarball created by gbp from - # the upstream git tag — the normal gbp path handles those correctly. - if [[ -f "./debian/watch" ]] && grep -q "qartifactory" "./debian/watch"; then - echo "ℹ️ debian/watch (Artifactory) found — fetching orig tarball via uscan" - uscan --destdir "$WORK_DIR" --download-current-version - GBP_ORIG_FLAGS=(--git-no-create-orig) - else - GBP_ORIG_FLAGS=() - fi - - # Note: --source-option=--extend-diff-ignore is passed to gbp/dpkg-buildpackage - # during the -S step so the exclusion is baked into the .dsc; the sbuild - # invocation below does not need --dpkg-source-opt for the same reason. - gbp buildpackage -S -sa -d -nc \ - --git-builder=dpkg-buildpackage \ - --git-ignore-branch \ - --git-debian-branch=HEAD \ - --git-ignore-new \ - "${GBP_ORIG_FLAGS[@]}" \ - --source-option=--extend-diff-ignore=^\.github \ - -us -uc - - DSC_FILE="${WORK_DIR}/${PKG}_${VER}.dsc" - [ -f "$DSC_FILE" ] || { echo "ERROR: .dsc not found after gbp buildpackage -S"; exit 1; } - echo "ℹ️ Source package ready: $DSC_FILE" - - sbuild --no-clean-source \ - --host=arm64 \ - --build=${BUILD_ARCH} \ - --arch-all \ - --dist=${INPUTS_SUITE} \ - $lintian_flag \ - --build-dir "$BUILD_DIR_ABS" \ - --build-dep-resolver=apt \ - "$DSC_FILE" - fi - - RET=$? - - if (( RET == 0 )); then - echo "✅ Successfully built package" - else - BUILD_LOG=$(find "$BUILD_DIR_ABS" -maxdepth 1 -name "*.build" ! -type l) - - if [[ -n "$BUILD_LOG" ]]; then - cat "$BUILD_LOG" - echo "❌ Build failed, printed the full build log file" - else - echo "❌ Build failed, but no .build log file was found to print" - fi - - exit 1 - fi diff --git a/.github/actions/push_to_repo/action.yml b/.github/actions/push_to_repo/action.yml deleted file mode 100644 index fc86d2bc..00000000 --- a/.github/actions/push_to_repo/action.yml +++ /dev/null @@ -1,149 +0,0 @@ -name: Push Built Package To Repo If Need Be -description: | - This Github Actions pushes the newly built package to a repository - -inputs: - - distro-codename: - description: The distribution codename to build for. Ex noble, jammy, etc - required: true - - token: - description: PAT token - required: true - - force-override: - description: If the version of the package already exists, override it. - default: false - -runs: - using: "composite" - - steps: - - # TODO deal with the case where multiple packages are built - - name: Extract built package name - id: extract-built-package - shell: bash - run: | - changes_file=$(find ./build-area -maxdepth 1 -name '*.changes' | head -n 1) - - echo "Found changes file : $changes_file" - - cat "$changes_file" - - # This line extracts the primary binary name from a changes file through a series of text processing steps: - # 1. grep '^Binary:' - Searches for lines starting with "Binary:" - # 2. sed 's/^Binary: //' - Removes the "Binary: " prefix from matched lines - # 3. tr ' ' '\n' - Converts spaces to newlines (splits binary names into separate lines) - # 4. grep -v -- '-dev' - Filters out any development packages (lines containing "-dev") - # 5. head -n 1 - Selects the first remaining binary name - # Result is stored in the main_binary variable for later use - # If no non-dev package exists, fall back to the first package (including -dev) - - echo "Extracting the main binary name from the changes file" - main_binary=$(grep '^Binary:' "$changes_file" | sed 's/^Binary: //' | tr ' ' '\n' | grep -v -- '-dev' || true) - echo "Main binary name extracted (excluding -dev packages) : $main_binary" - - if [ -z "$main_binary" ]; then - echo "No non-dev package found, falling back to the first package (including -dev)" - main_binary=$(grep '^Binary:' "$changes_file" | sed 's/^Binary: //' | tr ' ' '\n' | head -n 1) - fi - - version=$(grep '^Version:' "$changes_file" | sed 's/^Version: //') - - echo "built_package_name=$main_binary" >> "$GITHUB_OUTPUT" - echo "built_package_version=$version" >> "$GITHUB_OUTPUT" - - echo "Built package name : $main_binary" - echo "Built package version : $version" - - - name : List All The Versions Of The Built Packages Contained In The Staging PPA - shell: bash - env: - INPUTS_DISTRO_CODENAME: ${{inputs.distro-codename}} - BUILT_PACKAGE_NAME: ${{steps.extract-built-package.outputs.built_package_name}} - run: | - set +e - ./qcom-build-utils/scripts/ppa_interface.py \ - --operation list-versions \ - --apt-config "deb [arch=arm64 trusted=yes] ${REPO_URL} ${INPUTS_DISTRO_CODENAME}/stable main" \ - --package-name ${BUILT_PACKAGE_NAME} - - RET=$? - set -e - - - name: Check If Need To Upload To Repo - id: check-version - shell: bash - env: - INPUTS_DISTRO_CODENAME: ${{inputs.distro-codename}} - INPUTS_FORCE_OVERRIDE: ${{inputs.force-override}} - BUILT_PACKAGE_NAME: ${{steps.extract-built-package.outputs.built_package_name}} - BUILT_PACKAGE_VERSION: ${{steps.extract-built-package.outputs.built_package_version}} - run: | - echo "Checking if the repo already contains the built version" - - set +e - ./qcom-build-utils/scripts/ppa_interface.py \ - --operation contains-version \ - --apt-config "deb [arch=arm64 trusted=yes] ${REPO_URL} ${INPUTS_DISTRO_CODENAME}/stable main" \ - --package-name ${BUILT_PACKAGE_NAME} \ - --version ${BUILT_PACKAGE_VERSION} - - RET=$? - set -e - - echo "do_upload=true" >> $GITHUB_OUTPUT - - if [[ "$RET" == "0" && "${INPUTS_FORCE_OVERRIDE}" == "false" ]]; then - echo "Package version already exists in the repo and force-override is set to false. We are therefore done here." - echo "do_upload=false" >> $GITHUB_OUTPUT - elif [[ "$RET" == "0" && "${INPUTS_FORCE_OVERRIDE}" == "true" ]]; then - echo "Package version already exists in the repo, but force-override is set to true. Proceeding to override the package" - else - echo "The package version does not exist in the repo. Proceeding to uploat it." - fi - - - name: Checkout Staging Repo - if: steps.check-version.outputs.do_upload == 'true' - uses: actions/checkout@v5 - with: - repository: ${{env.REPO_NAME}} - ref: main - token: ${{inputs.token}} - path: ./pkg-oss-staging-repo - fetch-depth: 1 - - - name: Upload Debian Packages To Staging Repo - if: steps.check-version.outputs.do_upload == 'true' - shell: bash - env: - INPUTS_DISTRO_CODENAME: ${{inputs.distro-codename}} - INPUTS_TOKEN: ${{inputs.token}} - BUILT_PACKAGE_NAME: ${{steps.extract-built-package.outputs.built_package_name}} - BUILT_PACKAGE_VERSION: ${{steps.extract-built-package.outputs.built_package_version}} - run: | - ./qcom-build-utils/scripts/ppa_organizer.py --build-dir ./build-area --output-dir ./pkg-oss-staging-repo/pool/${INPUTS_DISTRO_CODENAME}/stable/main - - cd ./pkg-oss-staging-repo - - PPA_PACKAGES_FILE_REPO_PATH=dists/${INPUTS_DISTRO_CODENAME}/stable/main/binary-arm64 - - dpkg-scanpackages --multiversion pool/${INPUTS_DISTRO_CODENAME} > $PPA_PACKAGES_FILE_REPO_PATH/Packages - dpkg-scanpackages --type ddeb --multiversion pool/${INPUTS_DISTRO_CODENAME} >> $PPA_PACKAGES_FILE_REPO_PATH/Packages - - gzip -k -f $PPA_PACKAGES_FILE_REPO_PATH/Packages - - cat $PPA_PACKAGES_FILE_REPO_PATH/Packages - - git add . - - git config user.name "GitHub Service Bot" - git config user.email "githubservice@qti.qualcomm.com" - - git commit -s -m "Uploaded Package ${BUILT_PACKAGE_NAME} at version ${BUILT_PACKAGE_VERSION} for distro ${INPUTS_DISTRO_CODENAME}" - - git remote set-url origin https://x-access-token:${INPUTS_TOKEN}@github.com/${REPO_NAME}.git - - git push origin diff --git a/AGENTS.md b/AGENTS.md index 03ad4010..a53371ff 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -1,45 +1,54 @@ -# qcom-build-utils — Agent Guidelines +# qcom-build-utils - Agent Guidelines ## Purpose -`qcom-build-utils` hosts shared build utilities and composite actions for the -Qualcomm Linux ecosystem. +`qcom-build-utils` no longer owns package lifecycle CI assets. -Package lifecycle reusable workflows were moved to -`qualcomm-linux/pkg-infra/qli-ci`. +The following were migrated to `qualcomm-linux/qli-ci`: + +- reusable package workflows +- composite package actions +- shared package promotion/build scripts + +Use this repository for platform build helpers only. ## Current Scope -- Composite GitHub actions under `.github/actions/` -- Build/helper scripts under `scripts/` -- Platform build helpers under `kernel/`, `bootloader/`, `rootfs/`, and `flash/` -- Utility documentation under `docs/` +Primary maintained paths: + +- `kernel/` +- `bootloader/` +- `rootfs/` +- `flash/` +- repository metadata and issue templates under `.github/` -## Out of Scope +## Source of Truth for Package CI -Do not reintroduce package lifecycle reusable workflows in this repository. -Those belong in `qli-ci`, including: +For package lifecycle workflow behavior, use: -- pkg build/release/promote/upstream-pr reusable workflows -- package workflow templates (`.github/pkg-workflows/*`) -- package workflow sync automation +- `qualcomm-linux/qli-ci` for reusable workflows and package helper scripts +- `qualcomm-linux/debusine-action` for Debusine implementation details -## Workflow Ownership Model +## Do Not Reintroduce -- `qli-ci` is the package workflow source of truth for `pkg-*` repositories. -- `debusine-action` owns Debusine-specific implementation details. -- `qcom-build-utils` provides lower-level reusable tools and scripts. +Unless there is an explicit design decision, do not add back: + +- `.github/workflows/pkg-*.yml` reusable workflow definitions +- `.github/actions/` package composite actions +- `scripts/` package CI helper scripts ## Editing Guidance -- Prefer small, explicit changes focused on utilities/actions here. -- If a change affects package workflow orchestration, implement it in `qli-ci`. -- Keep references and examples aligned with the new ownership model. +When the request is about package promotion/build/release CI wiring, work in +`qli-ci` (and `debusine-action` when Debian Debusine internals are involved), +not here. + +For this repo, keep changes focused on platform build helpers. ## Validation Expectations -For utility/action changes in this repo: +For platform-helper changes: -1. validate touched scripts/actions locally where possible -2. check docs/examples for stale references -3. validate downstream caller impact only if action interfaces changed +1. run script-level checks locally where applicable +2. verify updated docs/reference paths are consistent +3. validate in the consuming repository before merge when possible diff --git a/README.md b/README.md index 0f3f0d53..0489ca9b 100644 --- a/README.md +++ b/README.md @@ -1,76 +1,64 @@ # qcom-build-utils -Shared build utilities and composite actions for Qualcomm Linux infrastructure. +This repository now hosts Qualcomm Linux platform build helpers only. -## Scope +## Migration Status -`qcom-build-utils` now focuses on reusable tooling and scripts for build systems. -Package lifecycle reusable workflows (`pkg-build`, `pkg-promote`, -`pkg-release`, `pkg-upstream-pr-build`) were moved to +Package CI assets were migrated out of `qcom-build-utils` to [`qualcomm-linux/qli-ci`](https://github.com/qualcomm-linux/qli-ci). -## Repository Structure +Migrated paths: -```text -qcom-build-utils/ -├── .github/ -│ └── actions/ -│ ├── abi_checker/ -│ ├── build_package/ -│ └── push_to_repo/ -├── scripts/ -├── kernel/ -├── bootloader/ -├── rootfs/ -├── flash/ -└── docs/ -``` - -## Composite Actions - -- `abi_checker`: ABI compatibility checks against prior published packages. -- `build_package`: Debian package build helpers based on gbp/sbuild flows. -- `push_to_repo`: Publish built packages and metadata to APT staging repos. - -## Scripts and Build Helpers - -The repository also carries utilities used by Qualcomm Linux build pipelines, -including kernel, bootloader, and rootfs build helpers. +- `scripts/` +- `.github/actions/` +- `.github/workflows/` -## Migration Note +## Current Repository Scope -If your package repository still references workflows under: +The maintained build helper content in this repository is: -- `qualcomm-linux/qcom-build-utils/.github/workflows/*` +- `kernel/` +- `bootloader/` +- `rootfs/` +- `flash/` -retarget it to: +## Package CI Source of Truth -- `qualcomm-linux/qli-ci/.github/workflows/*` +For package build, promote, release, and upstream PR validation workflows, use: -and use `qli-ci` workflow inputs (for example `qli-ci-ref`). +- [`qualcomm-linux/qli-ci`](https://github.com/qualcomm-linux/qli-ci) +- [`qualcomm-linux/debusine-action`](https://github.com/qualcomm-linux/debusine-action) -## Documentation +## Repository Layout -See [`docs/`](docs/) for action and scripting documentation. +```text +qcom-build-utils/ +|- .github/ +| |- ISSUE_TEMPLATE/ +| |- PULL_REQUEST_TEMPLATE/ +|- bootloader/ +|- docs/ +|- flash/ +|- kernel/ +|- rootfs/ +|- AGENTS.md +|- CONTRIBUTING.md +|- LICENSE.txt +`- README.md +``` ## Related Repositories - [`qualcomm-linux/qli-ci`](https://github.com/qualcomm-linux/qli-ci) - package reusable workflows and templates. - [`qualcomm-linux/debusine-action`](https://github.com/qualcomm-linux/debusine-action) - Debusine implementation details. +- [`qualcomm-linux/pkg-example`](https://github.com/qualcomm-linux/pkg-example) - [`qualcomm-linux/docker-pkg-build`](https://github.com/qualcomm-linux/docker-pkg-build) - local/containerized package build reference. - -## Branches - -`main` is the primary development branch. ## Contributing -See [CONTRIBUTING.md](CONTRIBUTING.md). +See [CONTRIBUTING.md](CONTRIBUTING.md) for contribution requirements. ## License -qcom-build-utils is licensed under [BSD-3-Clause](https://spdx.org/licenses/BSD-3-Clause.html). +Licensed under the [BSD-3-Clause License](https://spdx.org/licenses/BSD-3-Clause.html). See [LICENSE.txt](LICENSE.txt). diff --git a/docs/README.md b/docs/README.md index ef6308c2..35972481 100644 --- a/docs/README.md +++ b/docs/README.md @@ -1,20 +1,15 @@ # qcom-build-utils Documentation -This directory contains documentation for the utilities and composite actions -that remain in `qcom-build-utils`. +`qcom-build-utils` no longer hosts package reusable workflows, package +composite actions, or package CI scripts. -## Documentation Index +Those assets were migrated to: -1. [GitHub Actions](./github-actions.md) -2. [build_package action](./actions/build_package.md) -3. [abi_checker action](./actions/abi_checker.md) -4. [push_to_repo action](./actions/push_to_repo.md) -5. [build_container action](./actions/build_container.md) +- [`qualcomm-linux/qli-ci`](https://github.com/qualcomm-linux/qli-ci) +- [`qualcomm-linux/debusine-action`](https://github.com/qualcomm-linux/debusine-action) -## Ownership Notes +Use this `docs/` tree for platform-helper documentation that remains in this +repository. -Package lifecycle reusable workflows and package workflow templates were moved to -`qualcomm-linux/qli-ci`. - -If you are looking for pkg workflow integration docs, use the documentation in -`qli-ci` and package-repo guidance in `pkg-workspace`. +For package workflow architecture and caller integration guidance, refer to the +`qli-ci` documentation instead. diff --git a/docs/actions/abi_checker.md b/docs/actions/abi_checker.md deleted file mode 100644 index 59cb03e8..00000000 --- a/docs/actions/abi_checker.md +++ /dev/null @@ -1,110 +0,0 @@ -# abi_checker - -**Path**: `.github/actions/abi_checker/action.yml` - -**Purpose**: Checks the Application Binary Interface (ABI) compatibility of a newly built package against the previous version in the repository. This helps prevent unintentional API/ABI breakage. - -## Inputs - -| Input | Required | Default | Description | -|-------|----------|---------|-------------| -| `distro-codename` | Yes | - | Distribution codename (noble, jammy, etc.) | - -## Environment Variables Required - -- `REPO_URL`: APT repository URL to download previous package version -- `BUILT_PACKAGE_NAME`: Name of the built package (set by previous steps) - -## Process Flow - -```mermaid -flowchart TD - A[Action Called] --> B[Run deb_abi_checker.py] - B --> C[Download Previous Version from Repo] - C --> D{Package Found?} - D -->|No| E[First Build - No Previous Version] - D -->|Yes| F[Extract Both Packages] - F --> G[Compare Symbols and Headers] - G --> H[Analyze ABI Changes] - H --> I[Generate results.txt] - I --> J{Check Results} - J -->|No Diff| K[✅ ABI_NO_DIFF] - J -->|Compatible| L[⚠️ ABI_COMPATIBLE_DIFF] - J -->|Incompatible| M[⚠️ ABI_INCOMPATIBLE_DIFF] - J -->|Stripped| N[❌ ABI_STRIPPED_PACKAGE] - J -->|PPA Error| O[❌ PPA_ERROR] - E --> P[⚠️ PPA_PACKAGE_NOT_FOUND] - K --> Q[Check Version Increment] - L --> Q - M --> Q - P --> Q - N --> R[Fail Build] - O --> R - Q --> S{Version PASS?} - S -->|Yes| T[✅ Complete] - S -->|No| U[❌ Version Check Failed] -``` - -## Return Codes - -The ABI checker returns a bitmask of results: - -| Bit | Value | Code | Meaning | -|-----|-------|------|---------| -| - | 0 | `RETURN_ABI_NO_DIFF` | No ABI differences detected | -| 0 | 1 | `RETURN_ABI_COMPATIBLE_DIFF` | ABI changed but compatible | -| 1 | 2 | `RETURN_ABI_INCOMPATIBLE_DIFF` | ABI changed incompatibly | -| 2 | 4 | `RETURN_ABI_STRIPPED_PACKAGE` | Package is stripped (error) | -| 3 | 8 | `RETURN_PPA_PACKAGE_NOT_FOUND` | No previous version found | -| 4 | 16 | `RETURN_PPA_ERROR` | Repository access error | - -## Failure Conditions - -The action **fails** the build if: - -1. **Stripped Package** (bit 2 set): Package has no debug symbols - cannot verify ABI -2. **PPA Error** (bit 4 set): Unable to access the repository -3. **Version Not Incremented**: New version is not greater than repository version - -## Warning Conditions - -The action **warns** but continues if: - -1. **Compatible Diff** (bit 0 set): ABI changed but backward compatible -2. **Incompatible Diff** (bit 1 set): ABI changed incompatibly (should increment version) -3. **No Previous Package** (bit 3 set): First build of this package - -## ABI Comparison Details - -The checker analyzes: - -- **Exported symbols**: Functions and variables in shared libraries -- **Symbol versions**: Version information attached to symbols -- **Header files**: Public API definitions (for -dev packages) -- **SONAME**: Shared object version naming - -## Version Check - -After ABI checking, verifies that: - -- New package version > Repository version (for updates) -- Version increment is appropriate for ABI changes: - - Major version: Incompatible changes - - Minor version: Compatible additions - - Patch version: Bug fixes only - -## Usage Example - -```yaml -- name: Run ABI Check - uses: ./qcom-build-utils/.github/actions/abi_checker - with: - distro-codename: noble -``` - -## Notes - -- Requires `build-area/` directory with built packages -- Creates `results.txt` with detailed analysis -- Does not fail on ABI differences, only on fatal errors -- Intended to inform developers, not block builds automatically diff --git a/docs/actions/build_container.md b/docs/actions/build_container.md deleted file mode 100644 index f565087e..00000000 --- a/docs/actions/build_container.md +++ /dev/null @@ -1,113 +0,0 @@ -# build_container - -**Path**: `.github/actions/build_container/action.yml` - -**Purpose**: Builds Docker container images used for Debian package compilation. These containers include all necessary build tools and dependencies. - -## Inputs - -| Input | Required | Default | Description | -|-------|----------|---------|-------------| -| `arch` | Yes | - | Architecture to build for (`amd64` or `arm64`) | -| `push-to-ghcr` | No | `true` | Whether to push built image to GitHub Container Registry | -| `token` | Yes | - | GitHub PAT for registry authentication | -| `username` | Yes | - | Username for registry authentication | - -## Environment Variables Required - -- `QCOM_ORG_NAME`: Organization name (typically `qualcomm-linux`) -- `IMAGE_NAME`: Base image name (typically `pkg-builder`) - -## Process Flow - -```mermaid -flowchart TD - A[Action Called] --> B[Run docker_deb_build.py --rebuild] - B --> C[Build noble Container] - B --> D[Build questing Container] - C --> E[Checkout pkg-example] - D --> E - E --> F[Test Build: pkg-example for noble] - F --> G[Test Build: pkg-example for questing] - G --> H{push-to-ghcr enabled?} - H -->|Yes| I[Login to GHCR] - H -->|No| J[Skip Push] - I --> K[Push arch-noble image] - K --> L[Push arch-questing image] - L --> M[Complete] - J --> M -``` - -## Container Images Built - -For each architecture, two images are built: - -- `ghcr.io/qualcomm-linux/pkg-builder:{arch}-noble` -- `ghcr.io/qualcomm-linux/pkg-builder:{arch}-questing` - -## Build Process - -1. **Rebuild containers**: Uses `docker_deb_build.py --rebuild` to build from Dockerfiles -2. **Checkout test package**: Clones `pkg-example` repository -3. **Test noble build**: Builds example package for Ubuntu 24.04 (noble) -4. **Test questing build**: Builds example package for Ubuntu 25.04 (questing) -5. **Push to registry**: Uploads images to GHCR (if enabled) - -## Container Contents - -Each container includes: - -- **Base OS**: Ubuntu (noble or questing) -- **Build tools**: - - `sbuild` - Schroot-based package builder - - `git-buildpackage` (gbp) - Debian package build tool - - `dpkg-dev` - Debian package development tools - - `devscripts` - Debian developer scripts - - `lintian` - Debian package quality checker -- **Cross-compilation** (amd64 images): - - ARM64 cross-compilation toolchain - - QEMU for ARM64 emulation -- **Utilities**: - - Git, wget, curl - - Python for build scripts - -## Image Tags - -Images use the format: -``` -ghcr.io/{org}/{name}:{arch}-{distro} -``` - -Examples: -- `ghcr.io/qualcomm-linux/pkg-builder:amd64-noble` -- `ghcr.io/qualcomm-linux/pkg-builder:arm64-noble` -- `ghcr.io/qualcomm-linux/pkg-builder:amd64-questing` -- `ghcr.io/qualcomm-linux/pkg-builder:arm64-questing` - -## Testing Strategy - -Before pushing to GHCR, the action validates containers by: - -1. Building `pkg-example` package in each container -2. Verifying build succeeds for both noble and questing -3. Only pushing if both test builds succeed - -This ensures published containers are functional. - -## Usage Example - -```yaml -- name: Build Container Images - uses: ./.github/actions/build_container - with: - arch: arm64 - push-to-ghcr: true -``` - -## Notes - -- Built images are cached by Docker for faster subsequent builds -- `pkg-example` must build successfully before images are pushed -- Images are only pushed on non-PR events (push to main, schedule, manual) -- ARM64 images are built on self-hosted ARM64 runners for reliability -- Cross-compilation using buildx was attempted but had QEMU issues diff --git a/docs/actions/build_package.md b/docs/actions/build_package.md deleted file mode 100644 index 86bef7c7..00000000 --- a/docs/actions/build_package.md +++ /dev/null @@ -1,98 +0,0 @@ -# build_package - -**Path**: `.github/actions/build_package/action.yml` - -**Purpose**: Builds a Debian package using git-buildpackage (gbp) and sbuild. This action handles both native and cross-compilation builds. - -## Inputs - -| Input | Required | Default | Description | -|-------|----------|---------|-------------| -| `suite` | Yes | - | Distribution codename or Debian suite (noble, questing, trixie, etc.) | -| `pkg-dir` | Yes | - | Directory containing the Debian package source | -| `build-dir` | Yes | - | Directory where build artifacts will be placed | -| `run-lintian` | No | `false` | Whether to run lintian quality checks | - -## Process Flow - -```mermaid -flowchart TD - A[Action Called] --> B[Detect Builder Architecture
X64 → amd64
ARM64 → arm64] - B --> C[Prepare Workspace
Create build-dir] - C --> D{Source Format?} - D -->|quilt| E[Quilt Format OK] - D -->|native| F[Native Format OK] - D -->|unknown| G[Error: Unsupported] - E --> H{Check for Extra Repo} - F --> H - H -->|Available| I[Add pkg.qualcomm.com repo] - H -->|Not Available| J[Skip Extra Repo] - I --> K[Run gbp buildpackage] - J --> K - K --> L{Lintian Enabled?} - L -->|Yes| M[Run with --run-lintian] - L -->|No| N[Run with --no-run-lintian] - M --> O{Build Success?} - N --> O - O -->|Yes| P[Build Complete] - O -->|No| Q[Print Last 500 Lines of Log] - Q --> R[Exit with Error] -``` - -## Build Architecture Logic - -The action determines the build configuration based on the runner architecture: - -| Runner Arch | BUILD_ARCH | HOST_ARCH | Build Type | -|-------------|------------|-----------|------------| -| X64 (amd64) | amd64 | arm64 | Cross-compile | -| ARM64 | arm64 | arm64 | Native build | - -## Build Command - -The action runs git-buildpackage with sbuild: - -```bash -gbp buildpackage \ - --git-ignore-branch \ - --git-builder="sbuild --host=arm64 \ - --build=${BUILD_ARCH} \ - --dist=${suite} \ - ${lintian_flag} \ - --build-dir ../${build-dir} \ - --build-dep-resolver=apt \ - ${EXTRA_REPO}" -``` - -## Key Features - -- **Cross-compilation support**: Can build ARM64 packages on x86_64 hosts -- **Native builds**: Can build ARM64 packages on ARM64 hosts (faster) -- **Chroot isolation**: Uses sbuild with unshare mode for clean builds -- **Extra repository**: Automatically adds internal Qualcomm repo if available -- **Error handling**: Prints build log tail on failure for debugging -- **Source format detection**: Supports both quilt and native formats - -## Build Artifacts - -After successful build, the following artifacts are created in `build-dir`: - -- `*.deb` - Binary package files -- `*.ddeb` - Debug symbol packages -- `*.changes` - Package change description -- `*.buildinfo` - Build environment information -- `*.build` - Build log -- `*.dsc` - Debian source control file (for non-native) -- Source archives (for non-native packages) - -## Usage Example - -```yaml -- name: Build Debian Package - uses: ./qcom-build-utils/.github/actions/build_package - with: - suite: noble - pkg-dir: package-repo - build-dir: build-area - run-lintian: true -``` diff --git a/docs/actions/push_to_repo.md b/docs/actions/push_to_repo.md deleted file mode 100644 index 5adcc7ee..00000000 --- a/docs/actions/push_to_repo.md +++ /dev/null @@ -1,119 +0,0 @@ -# push_to_repo - -**Path**: `.github/actions/push_to_repo/action.yml` - -**Purpose**: Uploads built Debian packages to the staging APT repository if they don't already exist. Handles repository metadata updates. - -## Inputs - -| Input | Required | Default | Description | -|-------|----------|---------|-------------| -| `distro-codename` | Yes | - | Distribution codename (noble, jammy, etc.) | -| `token` | Yes | - | GitHub PAT for repository access | -| `force-override` | No | `false` | Override package if version already exists | - -## Environment Variables Required - -- `REPO_URL`: APT repository URL -- `REPO_NAME`: GitHub repository name (e.g., `qualcomm-linux/pkg-oss-staging-repo`) - -## Process Flow - -```mermaid -flowchart TD - A[Action Called] --> B[Extract Package Name and Version
from .changes file] - B --> C[List Versions in Repository] - C --> D[Check if Version Exists] - D --> E{Version Exists?} - E -->|No| F[do_upload = true] - E -->|Yes| G{force-override?} - G -->|Yes| F - G -->|No| H[do_upload = false
Skip Upload] - F --> I[Checkout pkg-oss-staging-repo] - I --> J[Copy .deb files to pool/] - J --> K[Run dpkg-scanpackages
Update Packages index] - K --> L[Run dpkg-scanpackages --type ddeb
Add debug packages] - L --> M[Compress Packages with gzip] - M --> N[Git add, commit, push] - N --> O[Upload Complete] - H --> P[Skip - Already in Repo] -``` - -## Repository Structure - -Packages are organized as: - -``` -pkg-oss-staging-repo/ -├── pool/ -│ └── {distro}/ -│ └── stable/ -│ └── main/ -│ ├── package_1.0-1_arm64.deb -│ ├── package_1.0-1_arm64.ddeb -│ └── ... -└── dists/ - └── {distro}/ - └── stable/ - └── main/ - └── binary-arm64/ - ├── Packages - └── Packages.gz -``` - -## Package Index Generation - -The action generates APT metadata: - -1. **Scan binary packages**: `dpkg-scanpackages --multiversion pool/{distro}` -2. **Scan debug packages**: `dpkg-scanpackages --type ddeb --multiversion pool/{distro}` -3. **Combine and compress**: Create `Packages` and `Packages.gz` - -## Upload Process - -1. **Extract metadata** from `.changes` file: - - Package name (first non-dev binary) - - Version number - -2. **Check repository**: - - Query if version already exists - - Decide whether to upload - -3. **Clone repository**: - - Shallow clone of `pkg-oss-staging-repo` - -4. **Copy packages**: - - Use `ppa_organizer.py` to copy `.deb` and `.ddeb` files - -5. **Update metadata**: - - Regenerate `Packages` index - - Compress with gzip - -6. **Commit and push**: - - Git commit with descriptive message - - Push to repository - -## Commit Message Format - -``` -Uploaded Package {PACKAGE_NAME} at version {VERSION} for distro {DISTRO} -``` - -## Usage Example - -```yaml -- name: Push to Repository - uses: ./qcom-build-utils/.github/actions/push_to_repo - with: - distro-codename: noble - force-override: false - token: ${{secrets.TOKEN}} -``` - -## Notes - -- Only pushes if version doesn't exist (unless `force-override: true`) -- Handles both binary (`.deb`) and debug (`.ddeb`) packages -- Automatically updates APT repository metadata -- Uses bot credentials for git commits -- Repository is immediately available after push (GitHub Pages) diff --git a/docs/github-actions.md b/docs/github-actions.md deleted file mode 100644 index 3b1a7931..00000000 --- a/docs/github-actions.md +++ /dev/null @@ -1,52 +0,0 @@ -# GitHub Actions - -This document describes the composite GitHub Actions in `qcom-build-utils`. - -## Overview - -Composite actions live under `.github/actions/` and provide reusable utility -steps that can be consumed by workflows in this repository or externally. - -## Available Actions - -1. [build_package](./actions/build_package.md) -2. [abi_checker](./actions/abi_checker.md) -3. [push_to_repo](./actions/push_to_repo.md) -4. [build_container](./actions/build_container.md) - -## Quick Reference - -| Action | Purpose | -|--------|---------| -| `build_package` | Build Debian packages with gbp/sbuild flows | -| `abi_checker` | Validate ABI compatibility against prior versions | -| `push_to_repo` | Publish built artifacts to APT staging repositories | -| `build_container` | Build and validate package-builder container images | - -## Common Patterns - -### Action Location - -Actions are referenced relative to a checkout of this repository: - -```yaml -uses: ./qcom-build-utils/.github/actions/{action_name} -``` - -### Error Handling - -Actions use strict shell modes and explicit return-code checks. - -### Output Indicators - -Status output conventions: - -- ✅ Success -- ❌ Fatal error -- ⚠️ Warning -- ℹ️ Information - -## Notes - -Package lifecycle reusable workflows are no longer hosted in -`qcom-build-utils`; they now live in `qualcomm-linux/qli-ci`. diff --git a/scripts/README.md b/scripts/README.md deleted file mode 100644 index 3276543e..00000000 --- a/scripts/README.md +++ /dev/null @@ -1,247 +0,0 @@ -# qcom-build-utils Scripts - -## Overview - -This directory contains utility scripts for building, organizing, and managing Debian packages for Qualcomm Linux platforms. These tools provide a streamlined workflow for package development, testing, and distribution. - -## Available Tools - -### 1. docker_deb_build.py - -The primary tool for building Debian packages in a containerized environment. It works on both ARM64 and x86_64 hosts, building natively on ARM64 and cross-compiling on x86_64. - -**Key Features:** -- Builds Debian packages inside Docker containers -- Supports multiple distributions (noble, questing) -- Automatic Docker image creation on first run -- Optional lintian checks for package quality -- Custom APT repository support - -**Usage:** -```bash -./scripts/docker_deb_build.py --source-dir --output-dir -``` - -**Options:** -- `--source-dir`: Path to the source directory containing debian package source (default: current directory) -- `--output-dir`: Path to the output directory for built packages (default: parent directory) -- `--distro`: Target distribution - `noble` or `questing` (default: noble) -- `--run-lintian`: Run lintian quality checks on the built package -- `--extra-repo`: Additional APT repository configuration - - Example: `'deb [arch=arm64 trusted=yes] http://pkg.qualcomm.com noble/stable main'` -- `--rebuild`: Rebuild the Docker image before building the package - -**Examples:** -```bash -# Basic package build -./scripts/docker_deb_build.py --source-dir ./my-package --output-dir ./build - -# Build with lintian checks -./scripts/docker_deb_build.py --source-dir ./my-package --run-lintian - -# Build with custom repository -./scripts/docker_deb_build.py --source-dir ./my-package \ - --extra-repo 'deb [arch=arm64 trusted=yes] http://pkg.qualcomm.com noble/stable main' - -# Rebuild Docker image and build package -./scripts/docker_deb_build.py --source-dir ./my-package --rebuild -``` - -**Pro Tip:** Create a shell alias for easier use: -```bash -# Add to ~/.bashrc -alias debb="/scripts/docker_deb_build.py" - -# Then use it simply as: -debb --source-dir ./my-package -``` - -### 2. deb_abi_checker.py - -ABI (Application Binary Interface) compatibility checker for Debian packages. - -**Key Features:** -- Compares two versions of a package for ABI changes -- Uses `abipkgdiff` from libabigail -- Detects incompatible changes -- Downloads old versions from PPA for comparison -- Generates detailed comparison reports - -**Usage:** -```bash -./scripts/deb_abi_checker.py --new-package-dir -``` - -**Options:** -- `--new-package-dir`: Directory containing new package (.deb, optional -dev.deb, optional -dbgsym.ddeb) -- `--apt-server-config`: APT server to download old package from - - Default: `'deb [arch=arm64 trusted=yes] http://pkg.qualcomm.com noble/stable main'` -- `--old-version`: Specific old version to compare against (optional, defaults to latest) -- `--delete-temp`: Delete temporary extracted folders after comparison -- `--result-file`: Path to save the comparison result file - -**Examples:** -```bash -# Compare against latest version from PPA -./scripts/deb_abi_checker.py --new-package-dir ./build/new-package - -# Compare against specific version -./scripts/deb_abi_checker.py \ - --new-package-dir ./build/new-package \ - --old-version 1.0-1 - -# Save results to file -./scripts/deb_abi_checker.py \ - --new-package-dir ./build/new-package \ - --result-file ./abi-report.txt -``` - -**Return Codes:** -- `0b00000` (0): No ABI differences detected -- `0b00001` (1): Compatible ABI changes detected -- `0b00010` (2): Incompatible ABI changes detected -- `0b00100` (4): Package is stripped (no debug symbols) -- `0b01000` (8): Old package not found in PPA -- `0b10000` (16): PPA error - -### 3. merge_debian_packaging_upstream - -Shell script for merging upstream changes into Debian packaging branch. - -**Key Features:** -- Merges upstream changes while preserving debian/ directory -- Also preserves .github/ directory -- Similar to `gbp-import-ref --merge-mode=replace` but with .github/ support - -**Prerequisites:** -- Debian packaging branch must be checked out -- Working tree must be clean -- Not in detached HEAD state - -**Usage:** -```bash -./scripts/merge_debian_packaging_upstream -``` - -**Example:** -```bash -# Merge upstream tag -./scripts/merge_debian_packaging_upstream v1.2.3 - -# Merge upstream branch -./scripts/merge_debian_packaging_upstream upstream/main -``` - -### 4. resolve_branch_family_suite.sh - -Helper used by reusable workflows to resolve `family`/`suite` from branch-like refs. - -**Rule:** -- Normalize refs like `refs/heads/*`, `refs/remotes/*`, and `origin/*` -- Split by `/` -- Take the last two fields as `/` -- Require `family` to be `debian` or `ubuntu` - -**Usage:** -```bash -./scripts/resolve_branch_family_suite.sh -``` - -**Output:** -```text -normalized_ref= -family= -suite= -``` - -## Common Workflows - -### Building a Single Package - -```bash -# 1. Build the package -./scripts/docker_deb_build.py --source-dir ./my-package --output-dir ./build - -# 2. Check ABI compatibility (optional) -./scripts/deb_abi_checker.py --new-package-dir ./build -``` - -### Setting Up a Development Environment - -```bash -# Create alias for quick access -echo 'alias debb="$(pwd)/scripts/docker_deb_build.py"' >> ~/.bashrc -source ~/.bashrc - -# Build Docker image once -debb --rebuild --source-dir ./some-package - -# Now you can quickly build packages -debb --source-dir ./package1 -debb --source-dir ./package2 --run-lintian -``` - -## Requirements - -### System Requirements -- **Operating System**: Linux (Ubuntu recommended) -- **Python**: 3.6 or later -- **Docker**: Required for docker_deb_build.py - - Docker daemon must be running - - User must have Docker permissions (member of `docker` group) or run with sudo - -### Python Dependencies -The scripts use these Python modules (all included in scripts/): -- `color_logger`: Colored logging output -- `helpers`: Helper functions for directory operations - -### External Tools -- **Docker**: For containerized builds (docker_deb_build.py) -- **libabigail** (`abipkgdiff`): For ABI checking (deb_abi_checker.py) - -## Docker Setup - -The first time you run `docker_deb_build.py`, it will automatically build the required Docker image from the Dockerfile in the `docker/` directory. The Dockerfiles are architecture and distribution specific: - -- `docker/Dockerfile.arm64.noble` - ARM64 build for Ubuntu Noble -- `docker/Dockerfile.arm64.questing` - ARM64 build for Ubuntu Questing -- `docker/Dockerfile.amd64.noble` - x86_64 build for Ubuntu Noble -- `docker/Dockerfile.amd64.questing` - x86_64 build for Ubuntu Questing - -To rebuild the Docker image: -```bash -./scripts/docker_deb_build.py --rebuild -``` - -## Troubleshooting - -### Docker Permission Issues - -If you get permission errors when running docker_deb_build.py: - -```bash -# Add your user to the docker group -sudo usermod -aG docker $USER - -# Start a new shell with updated group membership -newgrp docker - -# Or logout and login again -``` - -### Missing Dependencies - -If a script fails due to missing dependencies, ensure all required tools are installed: - -```bash -# Install Docker (Ubuntu/Debian) -sudo apt-get update -sudo apt-get install docker.io - -# Install libabigail for ABI checking -sudo apt-get install abigail-tools -``` - -## License - -qcom-build-utils is licensed under the [BSD-3-clause License](https://spdx.org/licenses/BSD-3-Clause.html). See [LICENSE.txt](LICENSE.txt) for the full license text. diff --git a/scripts/__init__.py b/scripts/__init__.py deleted file mode 100644 index 3a5a67d3..00000000 --- a/scripts/__init__.py +++ /dev/null @@ -1,5 +0,0 @@ -# Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. -# -# SPDX-License-Identifier: BSD-3-Clause-Clear - -"""Qcom Build Utilities""" diff --git a/scripts/color_logger.py b/scripts/color_logger.py deleted file mode 100644 index cc1315a5..00000000 --- a/scripts/color_logger.py +++ /dev/null @@ -1,73 +0,0 @@ -# Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. -# -# SPDX-License-Identifier: BSD-3-Clause-Clear - -""" -color_logger.py - -This module provides a color logger class, allowing users to log messages with colored text. -The class includes methods for logging messages at different levels, the same as the standard -python 'logging' mdule : debug, info, warning, error, and critical. - -Usage: - from color_logger import logger - - logger.debug('This is a debug message') - logger.info('This is an info message') - logger.warning('This is a warning message') - logger.error('This is an error message') - logger.critical('This is a critical message') -""" - -import logging -import datetime -import sys - -class ColorLogger: - LEVEL_STRING = { - logging.DEBUG: 'DEBG', - logging.INFO: 'INFO', - logging.WARNING: 'WARN', - logging.ERROR: 'ERR ', - logging.CRITICAL: 'CRIT' - } - - LEVEL_COLORS = { - logging.DEBUG: '\033[94m', #CYAN - logging.INFO: '\033[92m', #GREEN - logging.WARNING: '\033[93m', #YELLOW - logging.ERROR: '\033[91m', #RED - logging.CRITICAL: '\033[95m' #MAGENTA - } - - def __init__(self, name: str, level=logging.DEBUG): - self.logger = logging.getLogger(name) - self.logger.setLevel(level) - self.color_enabled = True - - handler = logging.StreamHandler() - handler.setFormatter(logging.Formatter('%(message)s')) - self.logger.addHandler(handler) - - def log(self, level, message): - reset = "\033[0m" - color = self.LEVEL_COLORS.get(level, "") - level_str = self.LEVEL_STRING.get(level, ' ') - colored_message = f"{color}{message}{reset}" if sys.stdout.isatty() else f"{message}" - timestamp = datetime.datetime.now().strftime("%H:%M:%S") - - self.logger.log(level, f"[{timestamp}] {level_str} : {colored_message if self.color_enabled else message}") - - def debug(self, msg): self.log(logging.DEBUG, msg) - def info(self, msg): self.log(logging.INFO, msg) - def warning(self, msg): self.log(logging.WARNING, msg) - def error(self, msg): self.log(logging.ERROR, msg) - def critical(self, msg): self.log(logging.CRITICAL, msg) - - def disable_color(self): - self.color_enabled = False - - def enable_color(self): - self.color_enabled = True - -logger = ColorLogger("BUILD") diff --git a/scripts/create_promotion_pr.py b/scripts/create_promotion_pr.py deleted file mode 100755 index 1a2e1ccb..00000000 --- a/scripts/create_promotion_pr.py +++ /dev/null @@ -1,149 +0,0 @@ -#!/usr/bin/env python3 -# -# Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. -# -# SPDX-License-Identifier: BSD-3-Clause-Clear - -""" -create_promotion_pr.py - -Helper script to generate and open the promotion PR. -""" -import subprocess -import argparse -import sys -import traceback -from color_logger import logger - -# This script is used to create the PR content of the promotion PR. Then, the actual PR is created using GitHub CLI -def parse_arguments(): - parser = argparse.ArgumentParser(description="Craft the content for a promotion PR and open it using GitHub CLI.") - - parser.add_argument("--base-branch", - required=False, - default="debian/qcom-next", - help="Base branch for the promotion PR.") - - parser.add_argument("--upstream-tag", - required=True, - help="Upstream tag corresponding to the version being promoted.") - - parser.add_argument("--normalized-version", - required=True, - help="Normalized version of the upstream project.") - - parser.add_argument("--promotion-version", - required=True, - help="Full Debian changelog version used for the promotion (for example 1.2.3-1 for Debian-bound branches, or 1.2.3-0qli1 for qli branches). This is also the suffix of the PR branch name debian/pr/.") - - parser.add_argument("--promotion-pr-branch", - required=False, - default="", - help="Promotion PR branch name (for example debian/pr/1.2.3-0qli1_). If omitted, defaults to debian/pr/.") - - parser.add_argument("--promoted-upstream-tag", - required=True, - help="Resolved promoted upstream tag created in the packaging repository (for example, upstream/1.2.3 or upstream/v1.2.3).") - - args = parser.parse_args() - - return args - -def create_pr_title(normalized_version: str) -> str: - return f"Promote to {normalized_version}" - -def create_pr_body(base_branch: str, upstream_tag: str, promotion_version: str, promotion_pr_branch: str, promoted_upstream_tag: str) -> str: - return f""" -## Automated promotion PR - -This PR was generated by `pkg-promote` to move this package repo to upstream tag -`{upstream_tag}`. - -### Summary - -- Base branch: `{base_branch}` -- Upstream branch/tag prepared: `upstream/latest` / `{promoted_upstream_tag}` -- PR branch: `{promotion_pr_branch}` -- Changelog update: `{promotion_version}` (UNRELEASED) - -### Reviewer checklist - -1. Review the upstream merge and packaging diff. -2. Confirm the `Build` workflow is green. -3. If needed, push follow-up packaging fixes to this PR branch. -4. Merge with **Merge commit** (do not squash or rebase). - -### Flow - -```mermaid ---- -config: - themeVariables: - 'gitInv2': '#ff0000' -gitGraph: - parallelCommits: true - rotateCommitLabel: true ---- -gitGraph BT: - branch {base_branch} order: 1 - branch upstream-main order: 4 - branch upstream/latest order: 3 - checkout main - commit id: 'Unrelated history: workflows, doc' - checkout upstream-main - commit - checkout upstream-main - commit - commit id: 'release' tag: '{upstream_tag}' - checkout upstream/latest - commit id: 'previous stuff' - merge upstream-main id: 'Filtered .github/debian folders' tag: '{promoted_upstream_tag}' - checkout {base_branch} - commit - commit - commit - branch {promotion_pr_branch} order: 2 - merge upstream/latest id: 'Merged Upstream' - commit id: 'Changelog version update' type: HIGHLIGHT -``` -""" - -def main(): - args = parse_arguments() - - logger.debug(f"Print of the arguments: {args}") - - promotion_pr_branch = args.promotion_pr_branch or f"debian/pr/{args.promotion_version}" - - pr_title = create_pr_title(args.normalized_version) - pr_body = create_pr_body( - args.base_branch, - args.upstream_tag, - args.promotion_version, - promotion_pr_branch, - args.promoted_upstream_tag, - ) - - # Printing the pr body in a .md file for manual review: - with open("promotion_pr_body.md", "w") as pr_body_file: - pr_body_file.write(pr_body) - - pr_creation_command = [ - "gh", "pr", "create", - "--title", pr_title, - "--body-file", "promotion_pr_body.md", - "--base", args.base_branch, - "--head", promotion_pr_branch, - ] - - # Executing the PR creation command using GitHub CLI - subprocess.run(pr_creation_command, check=True) - - -if __name__ == "__main__": - try: - main() - except Exception as e: - logger.critical(f"Uncaught exception : {e}") - traceback.print_exc() - sys.exit(1) diff --git a/scripts/deb_abi_checker.py b/scripts/deb_abi_checker.py deleted file mode 100755 index 6e1e860a..00000000 --- a/scripts/deb_abi_checker.py +++ /dev/null @@ -1,824 +0,0 @@ -#!/usr/bin/env python3 - -# Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. -# -# SPDX-License-Identifier: BSD-3-Clause-Clear - -""" -deb_abi_checker.py: ABI Comparison Tool - -This script compares two Debian binary packages (.deb) to detect ABI (Application Binary Interface) changes. -It usses the abipkgdiff tool (Package-level ABI comparison) - -------------------------------------------------- - - Compares the entire old .deb and new .deb packages directly. - - Internally extracts and analyzes binary symbols and type information using libabigail. - - Reports any changes in ABI that may cause incompatibility (e.g., removed or modified symbols). - - Advantages: - - Simple interface: only requires two .deb files as input. - - Ideal for high-level package comparison. - - Limitations: - - Depends on symbol/debug info availability. - - Does not show per-library granularity. - -Usage: - - -Options: - --report-dir Directory to save logs (default: ./reports) - --keep-temp Preserve extracted .deb directories for inspection -""" - -import os -import sys -import subprocess -import shutil -import argparse -import glob -import re -import traceback -from helpers import create_new_directory -from color_logger import logger - -RETURN_ABI_NO_DIFF = 0b00000 -RETURN_ABI_COMPATIBLE_DIFF = 0b00001 -RETURN_ABI_INCOMPATIBLE_DIFF = 0b00010 -RETURN_ABI_STRIPPED_PACKAGE = 0b00100 -RETURN_PPA_PACKAGE_NOT_FOUND = 0b01000 -RETURN_PPA_ERROR = 0b10000 - -class ABI_DIFF_Result: - def __init__(self, package_name): - self.package_name = package_name - self.repo_name = None - - self.new_deb_name=None - self.new_dev_name=None - self.new_ddeb_name=None - self.new_deb_version=None - - self.old_deb_name=None - self.old_dev_name=None - self.old_ddeb_name=None - self.old_deb_version=None - - self.abi_pkg_diff_result = None - self.abi_pkg_diff_remark = None - self.abi_pkg_diff_version_check = None - self.abi_pkg_diff_output = None - -# package_name - result -global_checker_results: dict[str, ABI_DIFF_Result] = {} - -def produce_report(log_file=None): - - log = "ABI Check results\n\n" - - log += ("-" * 100 + "\n") - - for package_name, result in global_checker_results.items(): - log += f"Package Name: {package_name}\n" - log += f"Repository Name: {result.repo_name}\n" - log += f"New Package:\n" - log += f" - DEB Name: {result.new_deb_name}\n" - log += f" - DEV Name: {result.new_dev_name}\n" - log += f" - DDEB Name: {result.new_ddeb_name}\n" - log += f" - Version: {result.new_deb_version}\n" - log += f"Old Package:\n" - log += f" - DEB Name: {result.old_deb_name}\n" - log += f" - DEV Name: {result.old_dev_name}\n" - log += f" - DDEB Name: {result.old_ddeb_name}\n" - log += f" - Version: {result.old_deb_version}\n" - log += f"ABI Package Diff:\n" - log += f" - Result: {result.abi_pkg_diff_result}\n" - log += f" - Version: {result.abi_pkg_diff_version_check}\n" - log += f" - Remark: {result.abi_pkg_diff_remark}\n" - log += f" - Output: {"" if result.abi_pkg_diff_output is not None else result.abi_pkg_diff_output}\n" - if result.abi_pkg_diff_output is not None: - cmd = f"echo \"{result.abi_pkg_diff_output}\" | sed 's/^/ /'" - output = subprocess.run(cmd, capture_output=True, text=True, shell=True) - - log += f"{output.stdout}\n" - - log += ("-" * 100 + "\n") - - if log_file is not None: - with open(log_file, 'w') as f: - f.write(log) - - logger.debug(log) - -def parse_arguments(): - parser = argparse.ArgumentParser(description="Compare two .deb packages using abipkgdiff or abidiff.") - parser.add_argument("--apt-server-config", - default="deb [arch=arm64 trusted=yes] http://pkg.qualcomm.com noble/stable main", - help="APT server configuration to download the old package to compare against") - - parser.add_argument("--new-package-dir", - required=True, - help="Path to the folder containing the new package to compare. (.deb, optional -dev.deb, optional -dbgsym.ddeb)") - - parser.add_argument("--delete-temp", - action="store_true", - help="Keep temp extracted folders for debugging.") - - parser.add_argument("--old-version", - required=False, - help="Specific version of the old package to compare against. (optional)") - - parser.add_argument("--result-file", - required=False, - help="Path for the result file") - - args = parser.parse_args() - - return args - -def main(): - args = parse_arguments() - - logger.debug(f"args: {args}") - - if not os.path.isabs(args.new_package_dir): - args.new_package_dir = os.path.abspath(args.new_package_dir) - - print_debug_tree = True - - - - ret = single_repo_deb_abi_checker(args.new_package_dir, - args.apt_server_config, - True if args.delete_temp is False else False, - None if not args.old_version else args.old_version, - print_debug_tree=print_debug_tree) - - if args.result_file is not None: - if not os.path.isabs(args.result_file): - args.result_file = os.path.abspath(args.result_file) - - produce_report(args.result_file) - - sys.exit(ret) - -def multiple_repo_deb_abi_checker(package_dir, apt_server_config, keep_temp=True, specific_apt_version=None) -> int: - """ - Runs the ABI check in a folder containing multiple package folders. - - Note: For a single package, use the function single_repo_deb_abi_checker() - - Args: - package_dir (str): Path to the temporary directory containing the packages. - Must have a structure like the following, where the core deb package is placed alongside - its development and debug package: - . - └── my_package - ├── my_package_1.0.0_arm64.deb - ├── my_package-dbgsym_1.0.0_arm64.ddeb - └── my_package-dev_1.0.0_arm64.deb - - apt_server_config (str): APT server configuration to download the old package to compare against. - Must be in the format "deb [arch=arm64 trusted=yes] http://pkg.qualcomm.com noble/stable main". - - keep_temp (bool): Whether to keep the temporary directory after the comparison. - - specific_apt_version (str): Specific version of the old package to compare against. (optional) - - - Returns: - -------- - - bool: Aglomeration of bitwise return value of every repo - """ - - final_ret = 0 - - for folder in os.listdir(package_dir): - folder_path = os.path.join(package_dir, folder) - if os.path.isdir(folder_path): - - try: - final_ret |= single_repo_deb_abi_checker(folder_path, apt_server_config, keep_temp, specific_apt_version) - except Exception as e: - logger.critical(f"Function single_repo_deb_abi_checker threw an exception: {e}") - - traceback.print_exc() - sys.exit(-1) - - log_file = os.path.join(package_dir, "abi_checker.log") - - produce_report(log_file) - - return final_ret - -def single_repo_deb_abi_checker(repo_package_dir, apt_server_config, keep_temp=True, specific_apt_version=None, print_debug_tree=False) -> int: - """ - Runs the ABI check for all the packages in a single repo output directory - - Note: For running the ABI check accross multiple repo folders, use the function multiple_repo_deb_abi_checker(), which - will run the ABI check for all the packages in all the repo folders. - - Args: - repo_package_dir (str): Path to the directory where a repo has build its packages. This directory - would typically be named after the repo name. For example, if the repo is named "my_package", - then the directory would be named "my_package". - - Must have a structure like the following, where the core deb package is placed alongside - its development and debug package: - . - └── repo_package_dir - ├── my_package_1.0.0_arm64.deb - ├── my_package-dbgsym_1.0.0_arm64.ddeb - └── my_package-dev_1.0.0_arm64.deb - - Note: It is possible for a repo to produce multiple core packages, in which case the directory - would contain multiple core packages. For example: - . - └── repo_package_dir - ├── my_package_1.0.0_arm64.deb - ├── my_package_2.0.0_arm64.deb - ├── my_package-dbgsym_1.0.0_arm64.ddeb - ├── my_package-dbgsym_2.0.0_arm64.ddeb - └── my_package-dev_1.0.0_arm64.deb - └── my_package-dev_2.0.0_arm64.deb - - If this is the case, this function will handle all the core packages in the directory. - - apt_server_config (str): APT server configuration to download the old package to compare against. - Must be in the format "deb [arch=arm64 trusted=yes] http://pkg.qualcomm.com noble/stable main". - - keep_temp (bool): Whether to keep the temporary directory after the comparison. - - specific_apt_version (str): Specific version of the old package to compare against. (optional) - - Returns: - -------- - - bool: True if the package ABI diff was performed sucessfully, False otherwise. - Note that this does not mean that the ABI diff passed, only that it was performed successfully. - """ - - logger.debug(f"[ABI_CHECKER]/[SINGLE_REPO]: Checking {repo_package_dir}") - - basedir = os.path.basename(repo_package_dir) - - logger.debug(f"[ABI_CHECKER]/[SINGLE_REPO]: performing abi checking for repo '{basedir}'") - - if print_debug_tree: - tree_cmd = f"tree -a {repo_package_dir} | sed 's/^/ /'" - tree_output = subprocess.run(tree_cmd, capture_output=True, text=True, shell=True) - if tree_output.returncode == 0: - logger.debug(f"[ABI_CHECKER]/[SINGLE_REPO]: Content :\n{tree_output.stdout}") - else: - logger.error(f"[ABI_CHECKER]/[SINGLE_REPO]: Failed to run 'tree' command: {tree_output.stderr}") - - abi_check_temp_dir = os.path.join(repo_package_dir, "abi_check_tmp") - - create_new_directory(abi_check_temp_dir, delete_if_exists=True) # <-- !delete the directory if it already exists - - # Find the .deb file(s) in the abi_check_temp_dir that represents the core packages - # We filter out the -dev and -dbgsym packages as we are interested in building the list of core packages - # that are built from the repo. - deb_files = [f for f in os.listdir(repo_package_dir) if f.endswith('.deb') and '-dev' not in f and '-dbgsym' not in f] - - if not deb_files: - logger.warning(f"[ABI_CHECKER]/[SINGLE_REPO]: No .deb file found, nothing to compare, returning success") - return RETURN_ABI_NO_DIFF - - logger.debug(f"[ABI_CHECKER]/[SINGLE_REPO]: Found {len(deb_files)} package{"s" if len(deb_files) > 1 else ""}") - - final_ret = 0 - - for deb_file in deb_files: - logger.debug(f"[ABI_CHECKER]/[SINGLE_REPO]: core deb file detected: {deb_file}") - package_name = os.path.splitext(os.path.basename(deb_file))[0].split('_')[0] - logger.debug(f"[ABI_CHECKER]/[SINGLE_REPO]: package name: {package_name}") - - package_abi_check_temp_dir = os.path.join(abi_check_temp_dir, package_name) - create_new_directory(package_abi_check_temp_dir) - - global_checker_results[package_name] = ABI_DIFF_Result(package_name) - global_checker_results[package_name].repo_name = basedir - - # Run the single_package_abi_checker function for the package - ret = single_package_abi_checker(repo_package_dir=repo_package_dir, - package_abi_check_temp_dir=package_abi_check_temp_dir, - package_name=package_name, - package_file=deb_file, - apt_server_config=apt_server_config, - keep_temp=keep_temp, - specific_apt_version=specific_apt_version, - print_debug_tree=print_debug_tree) - - final_ret = final_ret | ret - - return final_ret - -def single_package_abi_checker(repo_package_dir, - package_abi_check_temp_dir, - package_name, - package_file, - apt_server_config, - keep_temp=True, - specific_apt_version=None, - print_debug_tree=False) -> int: - """ - Runs the ABI check in a folder containing a single package. - """ - - result = global_checker_results[package_name] - - logger.debug(f"[ABI_CHECKER]/{package_name}: running single_package_abi_checker") - - old_extract_dir = os.path.join(package_abi_check_temp_dir, "old") - new_extract_dir = os.path.join(package_abi_check_temp_dir, "new") - - create_new_directory(old_extract_dir) - create_new_directory(new_extract_dir) - - new_version = os.path.splitext(package_file)[0].split('_')[1] - logger.info(f"[ABI_CHECKER]/{package_name}: New package version: {new_version}") - - new_deb_path = os.path.join(repo_package_dir, package_file) - - result.new_deb_name = package_file - result.new_deb_version = new_version - - # -dev.deb package is optional, but if it exists, we need to extract it too - # The package name may contain the major version number at the end, but by canonical convention, dev packages shall not contain that - # major version, so deal with this to make sure the dev package not containing it is found - package_name_without_major = (package_name[:-1] if package_name[-1].isdigit() else package_name) - - - deb_dev_files = [f for f in os.listdir(repo_package_dir) if f.endswith('.deb') and package_name_without_major in f and "-dev" in f] - - if not deb_dev_files: - logger.warning(f"[ABI_CHECKER]/{package_name}: No -dev.deb package found") - new_dev_path = None - elif len(deb_dev_files) == 1: - logger.info(f"[ABI_CHECKER]/{package_name}: -dev.deb package found: {deb_dev_files[0]}") - new_dev_path = os.path.join(repo_package_dir, deb_dev_files[0]) - result.new_dev_name = deb_dev_files[0] - else: - deb_dev_file = [f for f in deb_dev_files if f"{package_name_without_major}-dev" in f] - if len(deb_dev_file) > 1: - logger.critical(f"[ABI_CHECKER]/{package_name}: Multiple -dev.deb files found") - result.new_dev_name = "ERROR : multiple detected" - return -1 - new_dev_path = os.path.join(repo_package_dir, deb_dev_file[0]) - result.new_dev_name = deb_dev_file[0] - - # -dbgsym.ddeb package is optional, but if it exists, we need to extract it too - - deb_ddeb_files = [f for f in os.listdir(repo_package_dir) if f.endswith('.ddeb') and f"{package_name}-dbgsym" in f] - - if not deb_ddeb_files: - logger.warning(f"[ABI_CHECKER]/{package_name}: No -dbgsym.ddeb package found") - new_ddeb_path = None - elif len(deb_ddeb_files) == 1: - logger.info(f"[ABI_CHECKER]/{package_name}: -dbgsym.ddeb debug package found: {deb_ddeb_files[0]}") - new_ddeb_path = os.path.join(repo_package_dir, deb_ddeb_files[0]) - result.new_ddeb_name = deb_ddeb_files[0] - else: - logger.critical(f"[ABI_CHECKER]/{package_name}: Multiple -dev-dbgsym.ddeb files found") - result.new_ddeb_name = "ERROR : multiple detected" - return False - - # Extract all the packages in the 'new' directory - extract_deb(new_deb_path, new_dev_path, new_ddeb_path, new_extract_dir) - - if print_debug_tree: - # Run the 'tree' command to list files in a tree structure - tree_cmd = f"tree -a {new_extract_dir} | sed 's/^/ /'" - tree_output = subprocess.run(tree_cmd, capture_output=True, text=True, shell=True) - if tree_output.returncode == 0: - logger.debug(f"[ABI_CHECKER]/{package_name}: Tree structure of new_extract_dir:\n{tree_output.stdout}") - else: - logger.error(f"[ABI_CHECKER]/{package_name}: Failed to run 'tree' command: {tree_output.stderr}") - - # ******* OLD DEB PACKAGE fetching ********************************************************* - - logger.debug(f"[ABI_CHECKER]/{package_name}: Fetching old deb package from APT server") - logger.debug(f"[ABI_CHECKER]/{package_name}: APT Server Config: {apt_server_config}") - - old_download_dir = os.path.join(package_abi_check_temp_dir, "old_download") - - create_new_directory(old_download_dir) - - apt_dir = os.path.join(old_download_dir, "apt") - create_new_directory(apt_dir) - - # Use apt-get to download the latest version of the package - if specific_apt_version is None: - logger.debug(f"[ABI_CHECKER]/{package_name}: Using apt-get to download the *latest* version of {package_name}") - else: - logger.warning(f"[ABI_CHECKER]/{package_name}: Using apt-get to download the *specific* version {specific_apt_version} of {package_name}") - - # Create a temporary sources.list file - temp_sources_list = os.path.join(apt_dir, "sources.list") - with open(temp_sources_list, "w") as f: - f.write(apt_server_config) - - cache_dir = os.path.join(apt_dir, "cache") - create_new_directory(cache_dir) - - opt = f" -o Dir::Etc::sourcelist={temp_sources_list}" - opt += f" -o Dir::Etc::sourceparts=/dev/null" - opt += f" -o Dir::State={cache_dir}" - opt += f" -o Dir::Cache={cache_dir}" - - # Update the package list - cmd = "apt-get update" + opt - - logger.debug(f"[ABI_CHECKER]/{package_name}: Running: {cmd}") - apt_ret = subprocess.run(cmd, cwd=old_download_dir, shell=True, capture_output=True) - if apt_ret.returncode != 0: - logger.critical(f"[ABI_CHECKER]/{package_name}: Failed to update package list: {apt_ret.stderr}") - return RETURN_PPA_ERROR - - # download the .deb package - pkg = package_name + (("=" + specific_apt_version) if specific_apt_version else "") - cmd = f"apt-get download {pkg}" + opt - apt_ret = subprocess.run(cmd, cwd=old_download_dir, shell=True, capture_output=True) - if apt_ret.returncode != 0: - logger.error(f"[ABI_CHECKER]/{package_name}: Failed to download {pkg}: {apt_ret.stderr}") - return RETURN_PPA_PACKAGE_NOT_FOUND - else: - logger.info(f"[ABI_CHECKER]/{package_name}: Downloaded {pkg}") - - # download the -dev.deb package - pkg = package_name_without_major + "-dev" + (("=" + specific_apt_version) if specific_apt_version else "") - cmd = f"apt-get download {pkg}" + opt - apt_ret = subprocess.run(cmd, cwd=old_download_dir, shell=True, capture_output=True) - if apt_ret.returncode != 0: - logger.warning(f"[ABI_CHECKER]/{package_name}: Failed to download {pkg}: {apt_ret.stderr}") - else: - logger.info(f"[ABI_CHECKER]/{package_name}: Downloaded {pkg}") - - # download the -dbgsym.deb package - pkg = package_name + "-dbgsym" + (("=" + specific_apt_version) if specific_apt_version else "") - cmd = f"apt-get download {pkg}" + opt - apt_ret = subprocess.run(cmd, cwd=old_download_dir, shell=True, capture_output=True) - if apt_ret.returncode != 0: - logger.warning(f"[ABI_CHECKER]/{package_name}: Failed to download {pkg}: {apt_ret.stderr}") - else: - logger.info(f"[ABI_CHECKER]/{package_name}: Downloaded {pkg}") - - - # Configure the old packages paths - old_deb_file = next((f for f in os.listdir(old_download_dir) if f.endswith('.deb') and '-dev' not in f), None) - if old_deb_file is None: - logger.critical(f"[ABI_CHECKER]/{package_name}: No .deb file found in '{old_download_dir}' that does not contain '-dev' in the name") - result.old_deb_name = "ERROR : None found" - raise Exception("No .deb file found in '{old_download_dir}' that does not contain '-dev' in the name") - - old_deb_path = os.path.join(old_download_dir, old_deb_file) - result.old_deb_name = old_deb_file - result.old_deb_version = os.path.splitext(old_deb_file)[0].split('_')[1] - - - old_version = os.path.splitext(os.path.basename(old_deb_path))[0].split('_')[1] - logger.info(f"[ABI_CHECKER]/{package_name}: Old package version: {old_version}") - result.old_version = old_version - - old_dev_file = next((f for f in os.listdir(old_download_dir) if f.endswith('.deb') and '-dev' in f), None) - if old_dev_file is None: - old_dev_path = None - logger.warning(f"[ABI_CHECKER]/{package_name}: No -dev.deb file that does contains '-dev' in the name") - else: - old_dev_path = os.path.join(old_download_dir, old_dev_file) - result.old_dev_name = old_dev_file - - old_ddeb_file = next((f for f in os.listdir(old_download_dir) if f.endswith('.ddeb') and '-dbgsym' in f), None) - if old_ddeb_file is None: - old_ddeb_path = None - logger.warning(f"[ABI_CHECKER]/{package_name}: No -dbgsym.ddeb file found that does contains '-dbgsym' in the name") - else: - old_ddeb_path = os.path.join(old_download_dir, old_ddeb_file) - result.old_ddeb_name = old_ddeb_file - - extract_deb(old_deb_path, old_dev_path, old_ddeb_path, old_extract_dir) - - if print_debug_tree: - # Run the 'tree' command to list files in a tree structure - tree_cmd = f"tree -a {old_extract_dir} | sed 's/^/ /'" - tree_output = subprocess.run(tree_cmd, capture_output=True, text=True, shell=True) - if tree_output.returncode == 0: - logger.debug(f"[ABI_CHECKER]: Tree structure of old_extract_dir:\n{tree_output.stdout}") - else: - logger.error(f"[ABI_CHECKER]: Failed to run 'tree' command: {tree_output.stderr}") - - # ******* ABI CHECKING ********************************************************************** - - report_dir = os.path.join(package_abi_check_temp_dir,"report") - - abidiff_result = compare_with_abipkgdiff(old_deb_path, old_dev_path, old_ddeb_path, - new_deb_path, new_dev_path, new_ddeb_path, - report_dir, include_non_reachable_types=True) - - return_value = 0 - - # The return value between abidiff and abipkgdiff has the same meaning, so we can use the same analysis - if abidiff_result != 0: - - cmd =f"cat {report_dir}/abipkgdiff_output.txt" - - log = subprocess.run(cmd, shell=True, capture_output=True, text=True) - result.abi_pkg_diff_output = log.stdout - - - # Analyze the first 4 bits of the return value - bit1 = (abidiff_result & 0b0001) - bit2 = (abidiff_result & 0b0010) >> 1 - bit3 = (abidiff_result & 0b0100) >> 2 - bit4 = (abidiff_result & 0b1000) >> 3 - - # Determine the overall result based on the bit analysis - if bit1: - logger.critical(f"[ABI_CHECKER]: abipkgdiff encountered an error") - result.abi_pkg_diff_result = "ERROR" - raise Exception("abipkgdiff encountered an error") - if bit2: - logger.error(f"[ABI_CHECKER]: abipkgdiff usage error. This has shown to be true for stripped packages") - result.abi_pkg_diff_result = "STRIPPED-PACKAGE" - return RETURN_ABI_STRIPPED_PACKAGE - if bit3: - result.abi_pkg_diff_result = "COMPATIBLE-DIFF" - logger.warning(f"[ABI_CHECKER]: abipkgdiff detected ABI changes") - - return_value = RETURN_ABI_COMPATIBLE_DIFF - - match = re.search(r"Functions changes summary:\s+(\d+)\s+Removed,\s+(\d+)\s+Changed,", result.abi_pkg_diff_output) - if match: - changed_count = int(match.group(2)) - if changed_count > 0: - abidiff_result |= 0b1000 - return_value = RETURN_ABI_INCOMPATIBLE_DIFF - result.abi_pkg_diff_result = "INCOMPATIBLE-DIFF" - logger.warning(f"[ABI_CHECKER]: Overriding to INCOMPATIBLE CHANGE since there are changed functions") - - if bit4: - # if bit 4 is set, bit 3 must be too, so this fallthrough is ok - result.abi_pkg_diff_result = "INCOMPATIBLE-DIFF" - logger.warning(f"[ABI_CHECKER]: abipkgdiff detected ABI ***INCOMPATIBLE*** changes.") - return_value = RETURN_ABI_INCOMPATIBLE_DIFF - - # Print the content of all the files in 'report_dir' - for filename in os.listdir(report_dir): - file_path = os.path.join(report_dir, filename) - if os.path.isfile(file_path): - with open(file_path, 'r') as file: - logger.debug(f"Content of {filename}:") - logger.warning(file.read()) - - - else: - result.abi_pkg_diff_result = "NO-DIFF" - logger.info(f"[ABI_CHECKER]/{package_name}: abipkgdiff did not find any differences between old and new packages") - return_value = RETURN_ABI_NO_DIFF - - msg = "[ABI_CHECKER]/{package_name}: Although, no {pkg} was found for the {version} package, interpret the results with caution" - - if old_dev_path is None: - logger.warning(msg.format(package_name=package_name, pkg="-dev.deb", version="old")) - if new_dev_path is None: - logger.warning(msg.format(package_name=package_name, pkg="-dev.deb", version="new")) - if old_dev_path is None or new_dev_path is None: - result.abi_pkg_diff_remark = "NO-DEV-PACKAGE" - - if old_ddeb_path is None: - logger.warning(msg.format(package_name=package_name, pkg="-dbgsym.ddeb", version="old")) - if new_ddeb_path is None: - logger.warning(msg.format(package_name=package_name, pkg="-dbgsym.ddeb", version="new")) - if old_ddeb_path is None or new_ddeb_path is None: - if result.abi_pkg_diff_remark is not None: - result.abi_pkg_diff_remark += ", NO-DBG-PACKAGE" - else: - result.abi_pkg_diff_remark = "NO-DBG-PACKAGE" - - - if not keep_temp: - logger.debug(f"[ABI_CHECKER]: Removing temporary directory {abi_check_temp_dir}") - shutil.rmtree(abi_check_temp_dir) - - result.abi_pkg_diff_version_check = analyze_abi_diff_result(old_version, new_version, abidiff_result) - - return return_value - -def extract_deb(deb_path, dev_path, ddeb_path, extract_dir): - """Extract the content of a .deb package and its .ddeb to a specified directory.""" - - if deb_path is None: - raise ValueError("deb_path cannot be None") - if not deb_path.endswith(".deb") or not os.path.isfile(deb_path): - raise ValueError(f"Invalid deb_path: {deb_path}. Expected a file with .deb extension") - - cmd = ["dpkg", "-x", deb_path, extract_dir] - subprocess.run(cmd, check=True) - - if dev_path is not None: - cmd = ["dpkg", "-x", dev_path, extract_dir] - subprocess.run(cmd, check=True) - - if ddeb_path is not None: - cmd = ["dpkg", "-x", ddeb_path, extract_dir] - subprocess.run(cmd, check=True) - -def compare_with_abipkgdiff(old_deb_path, old_dev_path, old_ddeb_path, - new_deb_path, new_dev_path, new_ddeb_path, - report_dir, include_non_reachable_types=False): - """Run abipkgdiff on two .deb packages and log the result.""" - - logger.debug("[ABI_CHECKER]/[ABI_PKG_DIFF] : Comparing with abipkgdiff tool") - - os.makedirs(report_dir, exist_ok=True) - log_path = os.path.join(report_dir, "abipkgdiff_output.txt") - - cmd = "abipkgdiff" - - if include_non_reachable_types: - logger.debug("[ABI_CHECKER]/[ABI_PKG_DIFF] : Using --non-reachable-types option") - cmd += " --non-reachable-types" - - if old_dev_path is not None and new_dev_path is not None: - cmd += f" --devel-pkg1 {old_dev_path} --devel-pkg2 {new_dev_path}" - else: - logger.warning("[ABI_CHECKER]/[ABI_PKG_DIFF]: One or both of the -dev packages are missing. Potentially missing on information") - - if old_ddeb_path is not None and new_ddeb_path is not None: - cmd += f" --debug-info-pkg1 {old_ddeb_path} --debug-info-pkg2 {new_ddeb_path}" - else: - logger.warning("[ABI_CHECKER]/[ABI_PKG_DIFF]: One or both of the -dbgsym.ddeb packages are missing. Potentially missing on information") - - cmd += f" {old_deb_path} {new_deb_path}" - - - logger.debug(f"[ABI_CHECKER]/[ABI_PKG_DIFF]: command: {cmd}") - - abidiff_output = subprocess.run(cmd, capture_output=True, text=True, shell=True) - - with open(log_path, "w") as f: - f.write(abidiff_output.stdout) - - rc = abidiff_output.returncode - - return rc - -def version_bumped(old_version, new_version, index): - """ - Checks if the major version has been increased. - - Args: - old_version (str): The old version string (e.g., "1.0.0"). - new_version (str): The new version string (e.g., "2.0.0"). - - Returns: - bool: True if the major version has been increased, False otherwise. - """ - if index not in ["major", "minor", "patch"]: - raise ValueError("Index must be one of 'major', 'minor', or 'patch'") - - # Remove the build number from the version strings, if present - old_version = old_version.split('-')[0] - new_version = new_version.split('-')[0] - - # Split the version strings into their components - old_version_parts = list(map(int, old_version.split('.'))) - new_version_parts = list(map(int, new_version.split('.'))) - - # Determine the index of the version part to check - if index == "major": - index = 0 - elif index == "minor": - index = 1 - elif index == "patch": - index = 2 - - # Check if the version part at the specified index has increased - if new_version_parts[index] > old_version_parts[index]: - return True - else: - return False - - -def extract_upstream_version(version): - match = re.match(r'^(\d+\.\d+\.\d+)', version) - return match.group(1) if match else version - - -def analyze_abi_diff_result(old_version, new_version, abidiff_result) -> str: - import re - - logger.debug(f"old_version: {old_version}") - logger.debug(f"new_version: {new_version}") - - # Keep the first part of the version, before the first '-', '+' or '~' - - old_version = extract_upstream_version(old_version) - new_version = extract_upstream_version(new_version) - - - logger.debug(f"old_version: {old_version}") - logger.debug(f"new_version: {new_version}") - - # Define a regular expression pattern for a major-minor-patch version - version_pattern = r"^\d+\.\d+\.\d+(-\d+)?$" - - # Check if old_version and new_version match the pattern - if not re.match(version_pattern, old_version): - raise ValueError(f"Invalid old version: {old_version}. Expected a string in the format 'major.minor.patch'") - - if not re.match(version_pattern, new_version): - raise ValueError(f"Invalid new version: {new_version}. Expected a string in the format 'major.minor.patch'") - - logger.debug("[ABI_CHECKER]/[RESULT]: Performing version analysis of the ABI diff result versus the versions") - - # If both versions are valid, proceed with the analysis - # For now, just print the versions and the result - logger.debug(f"[ABI_CHECKER]/[RESULT]: Old version: {old_version}") - logger.debug(f"[ABI_CHECKER]/[RESULT]: New version: {new_version}") - - if (abidiff_result & 0b0011): - raise ValueError("[ABI_CHECKER]/[RESULT]: ASSERT : this scenario should have already been dealt with") - - abi_change = True if (abidiff_result & 0b0100) else False - incompatible_abi_change = True if (abidiff_result & 0b1000) else False - - - if incompatible_abi_change and not abi_change: - raise ValueError("[ABI_CHECKER]/[RESULT]: ASSERT : impossible scenario, if incompatible is set, change has to be set too") - - major_bumped = version_bumped(old_version, new_version, "major") - minor_bumped = version_bumped(old_version, new_version, "minor") - patch_bumped = version_bumped(old_version, new_version, "patch") - - if incompatible_abi_change: # Incompatible change - logger.error(f"[ABI_CHECKER]/[RESULT]: INCOMPATIBLE change detected") - - if major_bumped: - result = "PASS : Major version increased" - logger.debug(f"[ABI_CHECKER]/[RESULT]: {result}") - logger.debug("[ABI_CHECKER]/[RESULT]: Increasing the major version for an incompatible ABI is what is required") - - elif minor_bumped: - result = "FAIL : Minor version increased, needed major increase" - logger.debug(f"[ABI_CHECKER]/[RESULT]: {result}") - logger.debug(f"[ABI_CHECKER]/[RESULT]: Increasing only the minor version for an incompatible ABI change is not enough") - - elif patch_bumped: - result = "FAIL : Patch version increased, needed major increase" - logger.debug(f"[ABI_CHECKER]/[RESULT]: {result}") - logger.debug(f"[ABI_CHECKER]/[RESULT]: Increasing only the patch version for an incompatible ABI change is not enough") - - else: - result = "FAIL : No version increase" - logger.debug(f"[ABI_CHECKER]/[RESULT]: {result}") - logger.debug(f"[ABI_CHECKER]/[RESULT]: Increasing the version number is required for an ABI change") - - elif abi_change: # Compatible change - logger.warning(f"[ABI_CHECKER]/[RESULT]: COMPATIBLE change detected") - - if major_bumped: - result = "PASS : Major version increased" - logger.debug(f"[ABI_CHECKER]/[RESULT]: {result}") - logger.warning(f"[ABI_CHECKER]/[RESULT]: Increasing the major version for a compatible ABI change was probably overkill, but at least it respects version increase") - - elif minor_bumped: - result = "PASS : Minor version increased" - logger.debug(f"[ABI_CHECKER]/[RESULT]: {result}") - logger.debug(f"[ABI_CHECKER]/[RESULT]: Increasing the minor version for a compatible ABI change is what is required") - - elif patch_bumped: - result = "FAIL : Patch version increased, needed minor increase" - logger.debug(f"[ABI_CHECKER]/[RESULT]: {result}") - logger.debug(f"[ABI_CHECKER]/[RESULT]: Increasing only the patch number while there is an ABI change, albeit compatible, is not enough") - - else: - result = "FAIL : No version increase" - logger.debug(f"[ABI_CHECKER]/[RESULT]: {result}") - logger.debug(f"[ABI_CHECKER]/[RESULT]: Increasing at least the minor version number is required for a compatible ABI change") - - else: # No change - logger.info(f"[ABI_CHECKER]/[RESULT]: No ABI change detected") - - if major_bumped: - result = "PASS : Major version increased" - logger.debug(f"[ABI_CHECKER]/[RESULT]: {result}") - logger.warning("[ABI_CHECKER]/[RESULT]: Increasing the major version when there is no ABI change is probably overkill, but at least it respects version increase") - - elif minor_bumped: - result = "PASS : Minor version increased" - logger.debug(f"[ABI_CHECKER]/[RESULT]: {result}") - logger.warning(f"[ABI_CHECKER]/[RESULT]: Increasing the minor version for a compatible ABI change is probably overkill, but at least it respects version increase") - - elif patch_bumped: - result = "PASS : Patch version increased" - logger.debug(f"[ABI_CHECKER]/[RESULT]: {result}") - logger.debug(f"[ABI_CHECKER]/[RESULT]: Increasing only the patch number while there is no ABI change seems reasonable") - - else: - result = "PASS : No version increase" - logger.debug(f"[ABI_CHECKER]/[RESULT]: {result}") - - return result - -if __name__ == "__main__": - main() diff --git a/scripts/helpers.py b/scripts/helpers.py deleted file mode 100644 index 40869cf4..00000000 --- a/scripts/helpers.py +++ /dev/null @@ -1,67 +0,0 @@ -# Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. -# -# SPDX-License-Identifier: BSD-3-Clause-Clear - -""" -helper.py - -This module provides utilities for managing Debian package builds and related operations. -It includes functions for executing shell commands, managing files and directories, -logging, and setting up a local APT server. -""" - -import os -import stat -import shlex -import random -import shutil -import subprocess -import glob -from pathlib import Path - -from color_logger import logger - -def cleanup_directory(dirname): - """ - Removes a directory and its contents. - - Args: - ----- - - dirname (str): The path to the directory to clean up. - - Raises: - ------- - - Exception: If an error occurs while trying to remove the directory. - """ - try: - if os.path.exists(dirname): - shutil.rmtree(dirname) - except Exception as e: - logger.error(f"Error cleaning directory {dirname}: {e}") - raise Exception(e) - -def create_new_directory(dirname, delete_if_exists=True): - """ - Creates a new directory, optionally deleting it if it already exists. - - Args: - ----- - - dirname (str): The path to the directory to create. - - delete_if_exists (bool): If True, deletes the directory if it already exists. - - Raises: - ------- - - SystemExit: If an error occurs while creating the directory. - """ - - try: - if os.path.exists(dirname): - # Check if the directory exists, if so delete it - if delete_if_exists: - cleanup_directory(dirname) - # Create the destination directory - os.makedirs(dirname, exist_ok=not delete_if_exists) - except Exception as e: - logger.error(f"Error creating directory {dirname}: {e}") - exit(1) - diff --git a/scripts/merge_debian_packaging_upstream b/scripts/merge_debian_packaging_upstream deleted file mode 100755 index 123e0b1a..00000000 --- a/scripts/merge_debian_packaging_upstream +++ /dev/null @@ -1,72 +0,0 @@ -#!/bin/sh -set -e - -# merge_debian_packaging_upstream: given a Debian packaging branch, merge the -# latest upstream changes from the upstream branch. -# -# Prerequisites: -# * The Debian packaging branch is checked out and we are not in a detached -# HEAD state -# * The working tree is clean and unmodified -# -# Provide the upstream commitish as the only argument. Usually this would be -# the tag of the latest upstream release. -# -# The merge commit created works like `gbp-import-ref --merge-mode=replace` -# except that .github/ is replaced as well as debian/. - -upstream="$1" - -if ! debian_ref="$(git symbolic-ref HEAD)"; then - echo "Function requires the Debian packaging branch to be checked out" >&2 - echo "Are you in a detached HEAD state?" >&2 - exit 1 -fi - -case "$debian_ref" in - refs/heads/*) - debian_branch=${debian_ref#refs/heads/} - ;; - *) - echo "HEAD ref not understood" >&2 - echo "Do you have the Debian packaging branch checked out?" >&2 - exit 2 - ;; -esac - -echo "Merging upstream changes from upstream tag $upstream into Debian packaging branch $debian_branch" - -if ! upstream_commit="$(git rev-parse --verify "$upstream^{commit}" 2>/dev/null)"; then - echo "Could not resolve upstream input '$upstream' to a commit" >&2 - exit 3 -fi - -git checkout "$upstream_commit" -git reset "$debian_ref" -- :/:debian :/:.github -tree=$(git write-tree) - -merge_commit_header="Merge '$upstream' into Debian packaging branch '$debian_branch'" - -merge_commit_body=$(cat < bool : - - command = "apt-get update" + OPT - - logger.debug(f"[PPA_INTERFACE]/{PACKAGE_NAME}: Running: {command}") - - apt_ret = subprocess.run(command, cwd=TEMP_DIR, shell=True, capture_output=True) - - if apt_ret.returncode != 0: - logger.critical(f"[PPA_INTERFACE]/{PACKAGE_NAME}: Failed to update package list: {apt_ret.stderr}") - return False - - logger.info("[PPA_INTERFACE]/{PACKAGE_NAME}: Successfuly ran apt-get update") - - return True - -def download_package() -> bool : - global PACKAGE_NAME - global PACKAGE_VERSION - global OPT - global TEMP_DIR - - logger.debug(f"[PPA_INTERFACE]/[DOWNLOAD]/{PACKAGE_NAME}: Downloading version = {PACKAGE_VERSION} ") - - package = PACKAGE_NAME + ("" if PACKAGE_VERSION == None else ("=" + PACKAGE_VERSION)) - - command = f"apt-get download {package}" + OPT - - logger.debug(f"[PPA_INTERFACE]/[DOWNLOAD]/{PACKAGE_NAME}: Running: {command}") - - - apt_ret = subprocess.run(command, cwd=TEMP_DIR, shell=True, capture_output=True) - - if apt_ret.returncode != 0: - logger.error(f"[PPA_INTERFACE]/[DOWNLOAD]/{PACKAGE_NAME}: Failed to download {package}: {apt_ret.stderr}") - return False - - logger.info(f"[PPA_INTERFACE]/[DOWNLOAD]/{PACKAGE_NAME}: Downloaded {package}:\n{apt_ret.stdout}") - - return True - -def list_versions() : - logger.debug(f"[PPA_INTERFACE]/[LIST_VERSIONS]/{PACKAGE_NAME}: Listing versions available to download") - - command = f"apt-cache policy {PACKAGE_NAME} {OPT}" - - apt_ret = subprocess.run(command, cwd=TEMP_DIR, shell=True, capture_output=True) - - if apt_ret.returncode != 0: - logger.debug("command failed") - logger.info(f"stdout :\n{apt_ret.stdout}") - logger.info(f"stderr :\n{apt_ret.stderr}") - sys.exit(1) - - logger.info(f"stdout :\n{apt_ret.stdout.decode()}") - - -def contains_version(version : str) -> bool : - logger.debug(f"[PPA_INTERFACE]/[CONTAINS_VERSION]/{PACKAGE_NAME}: Checking if PPA contains version : {version}") - - command = f"apt list -a {PACKAGE_NAME} {OPT}" - - apt_ret = subprocess.run(command, cwd=TEMP_DIR, shell=True, capture_output=True) - - if apt_ret.returncode != 0: - logger.debug("command failed") - logger.info(f"stdout :\n{apt_ret.stdout}") - logger.info(f"stderr :\n{apt_ret.stderr}") - sys.exit(1) - - logger.debug(f"apt list stdout:\n{apt_ret.stdout.decode()}") - - if version in apt_ret.stdout.decode(): - logger.info(f"Found version : {version}") - sys.exit(0) - - logger.warning(f"Did not find version : {version}") - sys.exit(1) - -def main(): - - global APT_CONFIG - global TEMP_DIR - global PACKAGE_NAME - global PACKAGE_VERSION - - args = parse_arguments() - - logger.debug(f"args: {args}") - - APT_CONFIG = args.apt_config - PACKAGE_NAME = args.package_name - PACKAGE_VERSION = args.version - - if not os.path.isabs(args.temp_dir): - args.temp_dir = os.path.abspath(args.temp_dir) - - TEMP_DIR = args.temp_dir - - setup() - - run_apt_update() - - match args.operation: - case "download": - download_package() - - case "list-versions": - list_versions() - - case "contains-version": - if not args.version: - logger.critical("Need to supply --version") - sys.exit(1) - contains_version(args.version) - - case _: - sys.exit(1) - - - ret = 0 - - sys.exit(ret) - -if __name__ == "__main__": - - try: - main() - except Exception as e: - logger.critical(f"Uncaught exception : {e}") - - traceback.print_exc() - - sys.exit(1) \ No newline at end of file diff --git a/scripts/ppa_organizer.py b/scripts/ppa_organizer.py deleted file mode 100755 index 547e5226..00000000 --- a/scripts/ppa_organizer.py +++ /dev/null @@ -1,173 +0,0 @@ -#!/usr/bin/env python3 -# Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. -# -# SPDX-License-Identifier: BSD-3-Clause-Clear - -""" -ppa_organizer.py - -Organizes the output packages from a build into an organized PPA structure - -A PPA will contain a dists folder for the Packages.gz files, and a pool folder for the actual content. - -Consider this example tree for an example package build folder with sbuild: - -build/ -├── libqcom-example_1.1.0_arm64-2025-08-27T23:32:19Z.build -├── libqcom-example_1.1.0_arm64.build -> libqcom-example_1.1.0_arm64-2025-08-27T23:32:19Z.build -├── libqcom-example_1.1.0_arm64.buildinfo -├── libqcom-example_1.1.0_arm64.changes -├── libqcom-example_1.1.0.dsc -├── libqcom-example1_1.1.0_arm64.deb -├── libqcom-example1-dbgsym_1.1.0_arm64.ddeb -└── libqcom-example-dev_1.1.0_arm64.deb - -Note that here there is only one 'libqcom-example' package built, but when sbuild builds a package, -the debian/control file can list more packages. - -The goal of the operation is to copy over the .dsc, .deb and .ddeb files : -├── libqcom-example_1.1.0.dsc -├── libqcom-example1_1.1.0_arm64.deb -├── libqcom-example1-dbgsym_1.1.0_arm64.ddeb -└── libqcom-example-dev_1.1.0_arm64.deb - -Into the PPA structure, like this example tree : -└── pool - └── noble - └── stable - └── main - └── libqcom-example - ├── libqcom-example_1.1.0.dsc - ├── libqcom-example1_1.1.0_arm64.deb - ├── libqcom-example1-dbgsym_1.1.0_arm64.ddeb - └── libqcom-example-dev_1.1.0_arm64.deb - -This script will extract the 'canonical' package name (ie, without the major number in it, in this case its 1) -and create a folder in the PPA structure for this package name, and copy over all the .deb/ddeb that correspondt to it. - -This operation will be done for all the 'canonical' package names because again, there can be multiple of this example package -alongside one another - -""" - -import os -import sys -import shutil -import argparse -import subprocess - -from color_logger import logger -from helpers import create_new_directory - -def parse_arguments(): - parser = argparse.ArgumentParser(description="Organizes the output packages from a folder into a PPA repo structure") - parser.add_argument("--build-dir", - required=True, - help="The build directory where the packages have been built (.deb/.ddeb)") - - parser.add_argument("--output-dir", - required=True, - help="The output directory where the packages will be organized. In the example from the doc, it would be [...]/pool/noble/stable/main folder") - - args = parser.parse_args() - - return args - - - -def reorganize(build_dir : str, output_dir : str): - - logger.debug(f"Organize files from build dir : {build_dir} into : {output_dir}") - - # Create a list of all the packages (.deb, -dev.deb, -dbgsym.ddeb) - files = os.listdir(build_dir) - - dsc_files = [f for f in files if f.endswith('.dsc') ] - deb_files = [f for f in files if f.endswith('.deb') and "-dev" not in f] - dev_files = [f for f in files if f.endswith('.deb') and "-dev" in f] - dbg_files = [f for f in files if f.endswith('.ddeb') and "-dbgsym" in f] - - - # Isolate all the canonical package names (i.e. remove the version and architecture from the filenames) - dsc_pkg_names = [f.split('_')[0] for f in dsc_files] - deb_pkg_names = [f.split('_')[0] for f in deb_files] - dev_pkg_names = [f.split('_')[0].removesuffix("-dev") for f in dev_files] - dbg_pkg_names = [f.split('_')[0].removesuffix("-dbgsym") for f in dbg_files] - - # Second pass to remove all the major version that often suffix the package names - # The norm is that packages that include the major in the deb name DO NOT include it in the dev - # this ensures we deal with root package name and not doubles when we combine the lists below - dsc_pkg_names = [(f[:-1] if f[-1].isdigit() else f) for f in dsc_pkg_names] - deb_pkg_names = [(f[:-1] if f[-1].isdigit() else f) for f in deb_pkg_names] - dev_pkg_names = [(f[:-1] if f[-1].isdigit() else f) for f in dev_pkg_names] - dbg_pkg_names = [(f[:-1] if f[-1].isdigit() else f) for f in dbg_pkg_names] - - package_names = list(set(dsc_pkg_names) | set(deb_pkg_names) | set(dev_pkg_names) | set(dbg_pkg_names)) - - # Important that the list be sorted from the longest package name to the shortest - # Starting with the longest and removing it from the _files lists ensures we deal - # properly specificaly with the edge case or qcom-adreno/qcom-adreno-cl where one - # package name is a substring of the other - package_names.sort(reverse=True, key=lambda x: len(x)) - - for package_name in package_names: - - output_dir_pkg = os.path.join(output_dir, package_name) - - # Do not delete if the directory exists, it may very well contain the same package, but with older versions - # We want to copy the newly built packages alongside the other versions - create_new_directory(output_dir_pkg, delete_if_exists=False) - - logger.debug(f"Re-organizing outputs of package: {package_name}") - - dsc_package = next((file for file in dsc_files if package_name in file), None) - deb_package = next((file for file in deb_files if package_name in file), None) - dev_package = next((file for file in dev_files if package_name in file), None) - dbg_package = next((file for file in dbg_files if package_name in file), None) - - if dsc_package is not None: - shutil.copy(os.path.join(build_dir, dsc_package), os.path.join(output_dir_pkg, dsc_package)) - logger.info(f'Copied {dsc_package} to {output_dir_pkg}') - dsc_files.remove(deb_package) - else: - logger.debug(f"No .dsc package found for {package_name}") - - - if deb_package is not None: - shutil.copy(os.path.join(build_dir, deb_package), os.path.join(output_dir_pkg, deb_package)) - logger.info(f'Copied {deb_package} to {output_dir_pkg}') - deb_files.remove(deb_package) - else: - logger.debug(f"No .deb package found for {package_name}") - - if dev_package is not None: - shutil.copy(os.path.join(build_dir, dev_package), os.path.join(output_dir_pkg, dev_package)) - logger.info(f'Copied {dev_package} to {output_dir_pkg}') - dev_files.remove(dev_package) - else: - logger.debug(f"No -dev.deb package found for {package_name}") - - if dbg_package is not None: - shutil.copy(os.path.join(build_dir, dbg_package), os.path.join(output_dir_pkg, dbg_package)) - logger.info(f'Copied {dbg_package} to {output_dir_pkg}') - dbg_files.remove(dbg_package) - else: - logger.debug(f"No -dbgsym.ddeb package found for {package_name}") - -def main(): - - args = parse_arguments() - - logger.debug(f"args: {args}") - - # Make sure to resolve relative paths to absolute - if not os.path.isabs(args.build_dir): - args.build_dir = os.path.abspath(args.build_dir) - - if not os.path.isabs(args.output_dir): - args.output_dir = os.path.abspath(args.output_dir) - - reorganize(args.build_dir, args.output_dir) - -if __name__ == "__main__": - main() diff --git a/scripts/resolve_branch_family_suite.sh b/scripts/resolve_branch_family_suite.sh deleted file mode 100755 index 79e73609..00000000 --- a/scripts/resolve_branch_family_suite.sh +++ /dev/null @@ -1,70 +0,0 @@ -#!/usr/bin/env bash -# Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. -# -# SPDX-License-Identifier: BSD-3-Clause-Clear - -# Resolve distro family/suite from a branch-like ref. -# The last two '/'-delimited fields are interpreted as "/". - -set -euo pipefail - -normalize_ref() { - local ref="$1" - - if [[ "$ref" == refs/heads/* ]]; then - ref="${ref#refs/heads/}" - fi - if [[ "$ref" == refs/remotes/* ]]; then - ref="${ref#refs/remotes/}" - fi - if [[ "$ref" == origin/* ]]; then - ref="${ref#origin/}" - fi - - printf '%s\n' "$ref" -} - -resolve_from_ref() { - local ref="$1" - local -n out_family="$2" - local -n out_suite="$3" - local -a ref_parts - - IFS='/' read -r -a ref_parts <<< "$ref" - if (( ${#ref_parts[@]} < 2 )); then - return 1 - fi - - out_family="${ref_parts[$((${#ref_parts[@]} - 2))]}" - out_suite="${ref_parts[$((${#ref_parts[@]} - 1))]}" - - case "$out_family" in - debian|ubuntu) - return 0 - ;; - *) - return 1 - ;; - esac -} - -main() { - if (( $# != 1 )); then - echo "Usage: $0 " >&2 - exit 2 - fi - - local input_ref="$1" - local normalized_ref family suite - - normalized_ref="$(normalize_ref "$input_ref")" - if ! resolve_from_ref "$normalized_ref" family suite; then - exit 1 - fi - - printf 'normalized_ref=%s\n' "$normalized_ref" - printf 'family=%s\n' "$family" - printf 'suite=%s\n' "$suite" -} - -main "$@"