From 2852f91efcdd1c715b2c82817c8cbeb6be0abe53 Mon Sep 17 00:00:00 2001 From: Marco Trevisan Date: Fri, 23 Feb 2024 02:42:46 +0100 Subject: [PATCH 01/23] ci(go): generate: Use go mod edit to install properly versioned tools Parse the tools using jq so that we can easily get the tool path and version and install the one explicitly requested --- gh-actions/go/generate/action.yaml | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/gh-actions/go/generate/action.yaml b/gh-actions/go/generate/action.yaml index 299b5afd..a3fdbe37 100644 --- a/gh-actions/go/generate/action.yaml +++ b/gh-actions/go/generate/action.yaml @@ -24,6 +24,22 @@ outputs: runs: using: "composite" steps: + - name: Set up jq + shell: bash + env: + DEBIAN_FRONTEND: noninteractive + run: | + echo "::group::Download jq" + set -eu + if [ "${{runner.os}}" = "Windows" ]; then + winget.exe install jqlang.jq --accept-source-agreements --accept-package-agreements --silent --verbose || true + else + SUDO=$(command -v sudo || true) + $SUDO apt update + $SUDO apt install jq + fi + echo "::endgroup::" + jq --version - name: Install tools and dependencies id: proto-deps working-directory: ${{ inputs.tools-directory }} @@ -31,7 +47,8 @@ runs: echo "::group::Install tools and dependencies" set -eu - tools=$(grep -o '_ ".*"' *.go | cut -d '"' -f 2) + tools=$(go mod edit --json | \ + jq -r '.Require[] | select(.Indirect!=true) | [.Path,.Version] | join("@")') needsProtoc=false for tool in ${tools}; do From 72d23743c4a6c2dae6ad1111b85384491df411fb Mon Sep 17 00:00:00 2001 From: Didier Roche Date: Sat, 24 Feb 2024 12:01:50 +0100 Subject: [PATCH 02/23] Revert "ci(go): generate: Use go mod edit to install properly versioned tools" This reverts commit f4c4e2b876698fa92b44b7061dc89e079cb98a85. --- gh-actions/go/generate/action.yaml | 19 +------------------ 1 file changed, 1 insertion(+), 18 deletions(-) diff --git a/gh-actions/go/generate/action.yaml b/gh-actions/go/generate/action.yaml index a3fdbe37..299b5afd 100644 --- a/gh-actions/go/generate/action.yaml +++ b/gh-actions/go/generate/action.yaml @@ -24,22 +24,6 @@ outputs: runs: using: "composite" steps: - - name: Set up jq - shell: bash - env: - DEBIAN_FRONTEND: noninteractive - run: | - echo "::group::Download jq" - set -eu - if [ "${{runner.os}}" = "Windows" ]; then - winget.exe install jqlang.jq --accept-source-agreements --accept-package-agreements --silent --verbose || true - else - SUDO=$(command -v sudo || true) - $SUDO apt update - $SUDO apt install jq - fi - echo "::endgroup::" - jq --version - name: Install tools and dependencies id: proto-deps working-directory: ${{ inputs.tools-directory }} @@ -47,8 +31,7 @@ runs: echo "::group::Install tools and dependencies" set -eu - tools=$(go mod edit --json | \ - jq -r '.Require[] | select(.Indirect!=true) | [.Path,.Version] | join("@")') + tools=$(grep -o '_ ".*"' *.go | cut -d '"' -f 2) needsProtoc=false for tool in ${tools}; do From b6025d3675286950a8b728d2c025ed0443eca011 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edu=20G=C3=B3mez=20Escandell?= Date: Tue, 6 Feb 2024 13:39:09 +0100 Subject: [PATCH 03/23] Make debian version compliant Fix warnings related to: - Version number - File too long - No contributor name - Extra whitespace around name --- gh-actions/common/build-debian/action.yml | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/gh-actions/common/build-debian/action.yml b/gh-actions/common/build-debian/action.yml index a912bfeb..fbbd08ff 100644 --- a/gh-actions/common/build-debian/action.yml +++ b/gh-actions/common/build-debian/action.yml @@ -51,10 +51,20 @@ runs: DEBIAN_FRONTEND=noninteractive sudo apt install -y devscripts echo "::endgroup::" - echo "::group::Append commit SHA to local version" + echo "::group::Create local version with commit and docker container" cd '${{ inputs.source-dir }}' - sanitized_docker=$( echo "${{ inputs.docker-image }}" | sed 's/://' ) - debchange --local "+${sanitized_docker}+${{ github.sha }}" "Github build. Job id: ${{ github.run_id }}. Attempt: ${{ github.run_number }}." + + # Sanitize the docker name so that it stick to debian policy + # https://www.debian.org/doc/debian-policy/ch-controlfields.html#version + sanitized_docker=$( echo "${{ inputs.docker-image }}" | sed -r 's/[^a-zA-Z0-9.+~]+/+/g' ) + + # Short commit to avoid "package-has-long-file-name" + commit=$(echo ${{ github.sha }} | cut -c1-8) + + export DEBFULLNAME="GitHub actions runner" + export DEBEMAIL="noreply@github.com" + debchange --local "~${sanitized_docker}+${commit}" "Github build. Run id: ${{ github.run_id }}. Run number: ${{ github.run_number }}." + echo "::endgroup::" echo "::group::Parsing name and version" From 8cf7a2d7b725acde803e9086b6206d3633265c12 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marco=20Trevisan=20=28Trevi=C3=B1o=29?= Date: Sun, 25 Feb 2024 23:34:42 +0100 Subject: [PATCH 04/23] workflows: Add workflow to test build-deb action We don't fully support source builts for non-native packages yet though, so ignore the error for now --- .github/workflows/test-build-deb.yaml | 87 +++++++++++++++++++++++++++ 1 file changed, 87 insertions(+) create mode 100644 .github/workflows/test-build-deb.yaml diff --git a/.github/workflows/test-build-deb.yaml b/.github/workflows/test-build-deb.yaml new file mode 100644 index 00000000..d31cbd93 --- /dev/null +++ b/.github/workflows/test-build-deb.yaml @@ -0,0 +1,87 @@ +name: Test build debian package + +on: + push: + branches: + - main + paths: + - gh-actions/common/build-debian/** + - .github/workflows/test-build-deb* + pull_request: + +env: + DEBIAN_FRONTEND: noninteractive + DEBCONF_NONINTERACTIVE_SEEN: true + +jobs: + build_native_deb: + name: Test build native debian package + runs-on: ubuntu-latest + outputs: + pkg-name: ${{ env.PKG_NAME }} + pkg-version: ${{ env.PKG_VERSION }} + + steps: + - name: Install dependencies + run: | + sudo apt update + sudo apt install ubuntu-dev-tools + + - name: Get and prepare package source + run: | + set -eu + + echo "::group::Get source" + pull-lp-source --download-only hello + dpkg-source -x hello*.dsc hello-src + rm -rf hello_* + mv -v hello-src/* . + echo "::endgroup::" + + echo "::group::Mark package as a native package" + echo "3.0 (native)" > debian/source/format + dch -v$(dpkg-parsechangelog -S Version | cut -f1 -d-).1 \ + "Mark as native package" + echo "::endgroup::" + + - name: Checkout code + uses: actions/checkout@v4 + with: + path: .source + + - name: Build package + uses: ./.source/gh-actions/common/build-debian + with: + docker-image: ubuntu:devel + + build_source_deb: + name: Test build quilt debian package + runs-on: ubuntu-latest + outputs: + pkg-name: ${{ env.PKG_NAME }} + pkg-version: ${{ env.PKG_VERSION }} + + steps: + - name: Install dependencies + run: | + sudo apt update + sudo apt install ubuntu-dev-tools + + - name: Get package source + run: | + set -eu + + pull-lp-source --download-only hello + dpkg-source -x hello*.dsc hello-src + + - name: Checkout code + uses: actions/checkout@v4 + with: + path: .source + + - name: Build package + uses: ./.source/gh-actions/common/build-debian + with: + source-dir: ./hello-src + docker-image: ubuntu:devel + continue-on-error: true From be6482d89e7a7d811b09496c3e5fef9ceedc6312 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marco=20Trevisan=20=28Trevi=C3=B1o=29?= Date: Mon, 26 Feb 2024 04:00:02 +0100 Subject: [PATCH 05/23] build-debian: Clarify what artifacts are for Artifacts could be exposed by various other composed jobs so make this clearer for binary packages too --- gh-actions/common/build-debian/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gh-actions/common/build-debian/action.yml b/gh-actions/common/build-debian/action.yml index fbbd08ff..50aa8f16 100644 --- a/gh-actions/common/build-debian/action.yml +++ b/gh-actions/common/build-debian/action.yml @@ -119,6 +119,6 @@ runs: - name: Upload artifacts uses: actions/upload-artifact@v4 with: - name: ${{ env.PKG_NAME }}_${{ env.PKG_VERSION }} + name: ${{ env.PKG_NAME }}_${{ env.PKG_VERSION }}-debian-packages path: ${{ env.BUILD_OUTPUT_DIR }}/ if-no-files-found: error From 615b45487a02b12b60bad9b8f4dec3f6faa26dad Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marco=20Trevisan=20=28Trevi=C3=B1o=29?= Date: Mon, 26 Feb 2024 02:06:18 +0100 Subject: [PATCH 06/23] build-debian: Use stricter selection on dsc file based on env --- gh-actions/common/build-debian/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gh-actions/common/build-debian/action.yml b/gh-actions/common/build-debian/action.yml index 50aa8f16..a71dba89 100644 --- a/gh-actions/common/build-debian/action.yml +++ b/gh-actions/common/build-debian/action.yml @@ -107,7 +107,7 @@ runs: echo "::group::Extract source package" BUILD_INPUT_DIR=$(realpath "${BUILD_INPUT_DIR}") cd ${{ env.SOURCE_OUTPUT_DIR }} - dpkg-source --extract *.dsc "${BUILD_INPUT_DIR}" + dpkg-source --extract ${{ env.PKG_NAME }}_${{ env.PKG_VERSION }}.dsc "${BUILD_INPUT_DIR}" cd - echo "::endgroup::" - name: Build package From 79874f45279ef0cb96d6a72ef1a0c04cf5b5281e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marco=20Trevisan=20=28Trevi=C3=B1o=29?= Date: Mon, 26 Feb 2024 02:50:21 +0100 Subject: [PATCH 07/23] build-debian: Use manual docker build for source package This allows us to have more control on the volumes we mount in docker and on the location of the source path. And permits us to properly mount the parent folder of the source that we need to access to read the .orig files for non-native packages. It comes as a little extra cost, but also it give us way more control on the builder instance. --- .github/workflows/test-build-deb.yaml | 1 - gh-actions/common/build-debian/action.yml | 38 ++++++++++++++++++----- 2 files changed, 30 insertions(+), 9 deletions(-) diff --git a/.github/workflows/test-build-deb.yaml b/.github/workflows/test-build-deb.yaml index d31cbd93..6827b789 100644 --- a/.github/workflows/test-build-deb.yaml +++ b/.github/workflows/test-build-deb.yaml @@ -84,4 +84,3 @@ jobs: with: source-dir: ./hello-src docker-image: ubuntu:devel - continue-on-error: true diff --git a/gh-actions/common/build-debian/action.yml b/gh-actions/common/build-debian/action.yml index a71dba89..a2f67fde 100644 --- a/gh-actions/common/build-debian/action.yml +++ b/gh-actions/common/build-debian/action.yml @@ -74,21 +74,43 @@ runs: echo "::endgroup::" echo "::group::Prepare source build" - echo SOURCE_OUTPUT_DIR="$( mktemp --directory --tmpdir=. )" >> $GITHUB_ENV + echo SOURCE_OUTPUT_DIR="$( mktemp --directory --tmpdir="${PWD}" )" >> $GITHUB_ENV echo "::endgroup::" + - name: Build source package - uses: jtdor/build-deb-action@v1 + uses: kohlerdominik/docker-run-action@v1.2.0 with: - source-dir: ${{ inputs.source-dir }} - artifacts-dir: ${{ env.SOURCE_OUTPUT_DIR }} - docker-image: ${{ inputs.docker-image }} - buildpackage-opts: --build=source - extra-build-deps: ca-certificates git - before-build-hook: | + image: ${{ inputs.docker-image }} + environment: | + DEBIAN_FRONTEND=noninteractive + volumes: ${{ github.workspace }}:${{ github.workspace }} + workdir: ${{ github.workspace }}/${{ inputs.source-dir }} + shell: bash + run: | + echo "::group::Update builder instance" + set -eu + + echo 'APT::Get::Assume-Yes "true";' > /etc/apt/apt.conf.d/90aptyes + apt update + apt dist-upgrade + echo "::endgroup::" + + echo "::group::Install build dependencies" + apt build-dep . + apt install git ca-certificates + echo "::endgroup::" + GITHUB_TOKEN="${{ inputs.token }}" if [ -n "${GITHUB_TOKEN}" ]; then git config --system url."https://api:${GITHUB_TOKEN}@github.com/".insteadOf "https://github.com/" fi + + echo "::group::Build debian source package" + dpkg-buildpackage -D -S --sanitize-env + echo "::endgroup::" + + mv -v ../"${{ env.PKG_NAME }}_"* "${{ env.SOURCE_OUTPUT_DIR }}" + - name: Set up package build shell: bash run: | From dc4a92e117d8545a40fc834257f9ef1b652f32ba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marco=20Trevisan=20=28Trevi=C3=B1o=29?= Date: Mon, 26 Feb 2024 02:53:45 +0100 Subject: [PATCH 08/23] build-debian: Expose the extra source build-deps as parameter In general when building sources we should not need anything else a part what is in build-depends, but for some packages we may need some extra packages such as ca-certificates or git. So, expose these values as extra-source-build-deps input. This option should actually be unset by default, but we don't yet not to break existing packages, but we should change it ASAP so that it depends on each repo needs. --- .github/workflows/test-build-deb.yaml | 1 + gh-actions/common/build-debian/action.yml | 15 ++++++++++++--- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test-build-deb.yaml b/.github/workflows/test-build-deb.yaml index 6827b789..817f5b0c 100644 --- a/.github/workflows/test-build-deb.yaml +++ b/.github/workflows/test-build-deb.yaml @@ -84,3 +84,4 @@ jobs: with: source-dir: ./hello-src docker-image: ubuntu:devel + extra-source-build-deps: '' diff --git a/gh-actions/common/build-debian/action.yml b/gh-actions/common/build-debian/action.yml index a2f67fde..8bba6d18 100644 --- a/gh-actions/common/build-debian/action.yml +++ b/gh-actions/common/build-debian/action.yml @@ -12,11 +12,16 @@ inputs: token: required: false description: If provided, used for git authentication in the source build + extra-source-build-deps: + description: A list of extra build dependencies required during source build. + required: false + # FIXME: this should default to '', but we don't want to break job depending on us for now + default: 'ca-certificates git' # The process: -# 1. We build the source package in a docker container with ca-certificates installed and thus, -# a useful internet connection. +# 1. We build the source package in a docker container. If ca-certificates are +# installed via extra-source-build-deps we can have a useful internet connection. # 2. We the extract the source package. # 3. We build the .deb from the source package, in a container without ca-certificates (unless it # is added as a build dependency), hence without a useful internet connection. @@ -97,7 +102,11 @@ runs: echo "::group::Install build dependencies" apt build-dep . - apt install git ca-certificates + if [ -n "${{ inputs.extra-source-build-deps }}" ]; then + # Install extra packages for build-deps, to allow downloading vendored sources + deps=(${{ inputs.extra-source-build-deps }}) + apt install ${deps[@]} + fi echo "::endgroup::" GITHUB_TOKEN="${{ inputs.token }}" From c00a6bd3e1e7587db5a15e9b1d27196b2b846d2d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marco=20Trevisan=20=28Trevi=C3=B1o=29?= Date: Mon, 26 Feb 2024 02:57:26 +0100 Subject: [PATCH 09/23] build-debian: Upload source package as artifact It may be useful for debugging purposes, so upload it once it's ready --- gh-actions/common/build-debian/action.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/gh-actions/common/build-debian/action.yml b/gh-actions/common/build-debian/action.yml index 8bba6d18..fc68715e 100644 --- a/gh-actions/common/build-debian/action.yml +++ b/gh-actions/common/build-debian/action.yml @@ -120,6 +120,13 @@ runs: mv -v ../"${{ env.PKG_NAME }}_"* "${{ env.SOURCE_OUTPUT_DIR }}" + - name: Uploading source packages + uses: actions/upload-artifact@v4 + with: + name: ${{ env.PKG_NAME }}_${{ env.PKG_VERSION }}-debian-source + path: ${{ env.SOURCE_OUTPUT_DIR }}/ + if-no-files-found: error + - name: Set up package build shell: bash run: | From 878368750f68b59e77a3e9ea39ecedaf313fa83c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marco=20Trevisan=20=28Trevi=C3=B1o=29?= Date: Mon, 26 Feb 2024 03:03:38 +0100 Subject: [PATCH 10/23] build-debian: Use manual docker run to build the debian packages For some reason using jtdor/build-deb-action@v1 does not work well with some rust builds we have, so re-implement it using some manual labor. It's not really much work and it will allow us more control. Other than, working authd builds. --- gh-actions/common/build-debian/action.yml | 43 ++++++++++++++++++----- 1 file changed, 34 insertions(+), 9 deletions(-) diff --git a/gh-actions/common/build-debian/action.yml b/gh-actions/common/build-debian/action.yml index fc68715e..4ff08cfa 100644 --- a/gh-actions/common/build-debian/action.yml +++ b/gh-actions/common/build-debian/action.yml @@ -134,26 +134,51 @@ runs: echo "::group::Create build input directory" # Appending /source because 'dpkg-source --extract' needs the output directory to be non-existent - BUILD_INPUT_DIR="$( mktemp --directory --tmpdir='.' )/source" + BUILD_INPUT_BASEDIR="$( mktemp --directory --tmpdir="${PWD}" )" + echo BUILD_INPUT_BASEDIR="${BUILD_INPUT_BASEDIR}" >> $GITHUB_ENV + BUILD_INPUT_DIR="${BUILD_INPUT_BASEDIR}/source" echo BUILD_INPUT_DIR="${BUILD_INPUT_DIR}" >> $GITHUB_ENV echo "::endgroup::" echo "::group::Create build output directory" - echo BUILD_OUTPUT_DIR="$( mktemp --directory --tmpdir='.' )" >> $GITHUB_ENV + echo BUILD_OUTPUT_DIR="$( mktemp --directory --tmpdir="${PWD}" )" >> $GITHUB_ENV echo "::endgroup::" echo "::group::Extract source package" - BUILD_INPUT_DIR=$(realpath "${BUILD_INPUT_DIR}") cd ${{ env.SOURCE_OUTPUT_DIR }} dpkg-source --extract ${{ env.PKG_NAME }}_${{ env.PKG_VERSION }}.dsc "${BUILD_INPUT_DIR}" - cd - echo "::endgroup::" - - name: Build package - uses: jtdor/build-deb-action@v1 + + - name: Build packages + uses: kohlerdominik/docker-run-action@v1.2.0 with: - artifacts-dir: ${{ env.BUILD_OUTPUT_DIR }} - source-dir: ${{ env.BUILD_INPUT_DIR }} - docker-image: ${{ inputs.docker-image }} + image: ${{ inputs.docker-image }} + environment: | + DEBIAN_FRONTEND=noninteractive + workdir: ${{ env.BUILD_INPUT_DIR }} + volumes: | + ${{ env.BUILD_INPUT_BASEDIR }}:${{ env.BUILD_INPUT_BASEDIR }} + ${{ env.BUILD_OUTPUT_DIR }}:${{ env.BUILD_OUTPUT_DIR }} + shell: bash + run: | + echo "::group::Update builder instance" + set -eu + + echo 'APT::Get::Assume-Yes "true";' > /etc/apt/apt.conf.d/90aptyes + apt update + apt dist-upgrade + echo "::endgroup::" + + echo "::group::Install build dependencies" + apt build-dep . + echo "::endgroup::" + + echo "::group::Build debian packages" + dpkg-buildpackage -D -b --sanitize-env + echo "::endgroup::" + + mv -v ../*"_${{ env.PKG_VERSION }}_"*.deb "${{ env.BUILD_OUTPUT_DIR }}" + - name: Upload artifacts uses: actions/upload-artifact@v4 with: From 7cf3c3c3f7472cc245eb7afbf7f33455da768500 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marco=20Trevisan=20=28Trevi=C3=B1o=29?= Date: Mon, 26 Feb 2024 03:05:31 +0100 Subject: [PATCH 11/23] build-debian: Build the package as user When building debian packages we should not have high privileges, so let's just use an unprivileged user to perform the binary build. It's not too needed for source builds, since those are not really doing tests or anything where having root privileges may lead to different behavior --- gh-actions/common/build-debian/action.yml | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/gh-actions/common/build-debian/action.yml b/gh-actions/common/build-debian/action.yml index 4ff08cfa..020a6bdb 100644 --- a/gh-actions/common/build-debian/action.yml +++ b/gh-actions/common/build-debian/action.yml @@ -169,12 +169,23 @@ runs: apt dist-upgrade echo "::endgroup::" + echo "::group::Create build user" + apt install adduser + apt-mark auto adduser + adduser --disabled-password --gecos "" builder + chown builder:builder .. -R + echo "::endgroup::" + + echo "::group::Cleanup unneeded packages" + apt autoremove + echo "::endgroup::" + echo "::group::Install build dependencies" apt build-dep . echo "::endgroup::" echo "::group::Build debian packages" - dpkg-buildpackage -D -b --sanitize-env + runuser -u builder -- dpkg-buildpackage -D -b --sanitize-env echo "::endgroup::" mv -v ../*"_${{ env.PKG_VERSION }}_"*.deb "${{ env.BUILD_OUTPUT_DIR }}" From e55e0e5415b54b17484c6fe5a8e23444c4c8e064 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marco=20Trevisan=20=28Trevi=C3=B1o=29?= Date: Mon, 26 Feb 2024 03:10:04 +0100 Subject: [PATCH 12/23] build-debian: Fully disable internet access during packages build Ensure that internet access is completely disabled for builder user when creating the binary package. This is something that we somewhat had implicitly for HTTPS only since ca-certificates were not installed. But let's ensure this in any case. --- gh-actions/common/build-debian/action.yml | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/gh-actions/common/build-debian/action.yml b/gh-actions/common/build-debian/action.yml index 020a6bdb..d406c451 100644 --- a/gh-actions/common/build-debian/action.yml +++ b/gh-actions/common/build-debian/action.yml @@ -23,8 +23,8 @@ inputs: # 1. We build the source package in a docker container. If ca-certificates are # installed via extra-source-build-deps we can have a useful internet connection. # 2. We the extract the source package. -# 3. We build the .deb from the source package, in a container without ca-certificates (unless it -# is added as a build dependency), hence without a useful internet connection. +# 3. We build the .deb from the source package, in a container without internet +# kind of internet connection. # # To help with debugging, here are the processes and the directories they takes place in: # @@ -153,6 +153,7 @@ runs: uses: kohlerdominik/docker-run-action@v1.2.0 with: image: ${{ inputs.docker-image }} + options: --cap-add=NET_ADMIN environment: | DEBIAN_FRONTEND=noninteractive workdir: ${{ env.BUILD_INPUT_DIR }} @@ -176,6 +177,13 @@ runs: chown builder:builder .. -R echo "::endgroup::" + echo "::group::Fully disable internet access for user" + apt install iptables + apt-mark auto iptables + iptables -A OUTPUT -m owner --uid-owner $(id -u builder) -d 127.0.0.1 -j ACCEPT + iptables -A OUTPUT -m owner --uid-owner $(id -u builder) -j DROP + echo "::endgroup::" + echo "::group::Cleanup unneeded packages" apt autoremove echo "::endgroup::" From 63e270e3fb975f72b3e44f4d074924ed627842cb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marco=20Trevisan=20=28Trevi=C3=B1o=29?= Date: Mon, 26 Feb 2024 03:11:43 +0100 Subject: [PATCH 13/23] build-debian: Add DEB_BUILD_OPTIONS input It allows to perform special builds (such as nocheck ones), so expose it in case, for example, one may want to build without running tests. --- gh-actions/common/build-debian/action.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/gh-actions/common/build-debian/action.yml b/gh-actions/common/build-debian/action.yml index d406c451..16497a63 100644 --- a/gh-actions/common/build-debian/action.yml +++ b/gh-actions/common/build-debian/action.yml @@ -17,6 +17,10 @@ inputs: required: false # FIXME: this should default to '', but we don't want to break job depending on us for now default: 'ca-certificates git' + DEB_BUILD_OPTIONS: + required: false + default: '' + description: The DEB_BUILD_OPTIONS environment variable that is used during builds. # The process: @@ -88,6 +92,7 @@ runs: image: ${{ inputs.docker-image }} environment: | DEBIAN_FRONTEND=noninteractive + DEB_BUILD_OPTIONS=${{ inputs.DEB_BUILD_OPTIONS }} volumes: ${{ github.workspace }}:${{ github.workspace }} workdir: ${{ github.workspace }}/${{ inputs.source-dir }} shell: bash @@ -156,6 +161,7 @@ runs: options: --cap-add=NET_ADMIN environment: | DEBIAN_FRONTEND=noninteractive + DEB_BUILD_OPTIONS=${{ inputs.DEB_BUILD_OPTIONS }} workdir: ${{ env.BUILD_INPUT_DIR }} volumes: | ${{ env.BUILD_INPUT_BASEDIR }}:${{ env.BUILD_INPUT_BASEDIR }} From a28fcbf59c4b391b5b018f32a4323c6e34d7b600 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marco=20Trevisan=20=28Trevi=C3=B1o=29?= Date: Mon, 26 Feb 2024 03:50:16 +0100 Subject: [PATCH 14/23] build-debian: Use docker container also to prepare the source package In this way we can use the proper distro version number --- gh-actions/common/build-debian/action.yml | 58 ++++++++++++++++------- 1 file changed, 42 insertions(+), 16 deletions(-) diff --git a/gh-actions/common/build-debian/action.yml b/gh-actions/common/build-debian/action.yml index 16497a63..754dcbb1 100644 --- a/gh-actions/common/build-debian/action.yml +++ b/gh-actions/common/build-debian/action.yml @@ -55,28 +55,50 @@ runs: run: | set -eu - echo "::group::Install devscripts" - DEBIAN_FRONTEND=noninteractive sudo apt update - DEBIAN_FRONTEND=noninteractive sudo apt install -y devscripts - echo "::endgroup::" - echo "::group::Create local version with commit and docker container" cd '${{ inputs.source-dir }}' - # Sanitize the docker name so that it stick to debian policy - # https://www.debian.org/doc/debian-policy/ch-controlfields.html#version - sanitized_docker=$( echo "${{ inputs.docker-image }}" | sed -r 's/[^a-zA-Z0-9.+~]+/+/g' ) - # Short commit to avoid "package-has-long-file-name" - commit=$(echo ${{ github.sha }} | cut -c1-8) + echo VERSION_REF=$(date +'%y%m%d').${{ github.run_number }}.$(echo ${{ github.sha }} | cut -c1-8) >> $GITHUB_ENV - export DEBFULLNAME="GitHub actions runner" - export DEBEMAIL="noreply@github.com" - debchange --local "~${sanitized_docker}+${commit}" "Github build. Run id: ${{ github.run_id }}. Run number: ${{ github.run_number }}." - + echo DEBFULLNAME="GitHub actions runner" >> $GITHUB_ENV + echo DEBEMAIL="noreply@github.com" >> $GITHUB_ENV echo "::endgroup::" + - name: Prepare source package + uses: kohlerdominik/docker-run-action@v2.0.0 + with: + image: ${{ inputs.docker-image }} + environment: | + DEBIAN_FRONTEND=noninteractive + DEBFULLNAME=${{ env.DEBFULLNAME }} + DEBEMAIL=${{ env.DEBEMAIL }} + volumes: ${{ github.workspace }}:${{ github.workspace }} + workdir: ${{ github.workspace }}/${{ inputs.source-dir }} + shell: bash + run: | + echo "::group::Update builder instance and install dependencies" + set -eu + + echo 'APT::Get::Assume-Yes "true";' > /etc/apt/apt.conf.d/90aptyes + apt update + apt install devscripts lsb-release + echo "::endgroup::" + + echo "::group::Update debian package changelog" + dch --local "~$(lsb_release -r -s)+git${{ env.VERSION_REF }}" \ + "Github build. Run id: ${{ github.run_id }}. Run number: ${{ github.run_number }}." + + dpkg-parsechangelog + echo "::endgroup::" + + - name: Parse package source info + shell: bash + run: | echo "::group::Parsing name and version" + set -eu + + cd '${{ inputs.source-dir }}' echo PKG_NAME="$( dpkg-parsechangelog --show-field source )" >> $GITHUB_ENV echo PKG_VERSION="$( dpkg-parsechangelog --show-field version )" >> $GITHUB_ENV cd - @@ -87,7 +109,7 @@ runs: echo "::endgroup::" - name: Build source package - uses: kohlerdominik/docker-run-action@v1.2.0 + uses: kohlerdominik/docker-run-action@v2.0.0 with: image: ${{ inputs.docker-image }} environment: | @@ -114,6 +136,10 @@ runs: fi echo "::endgroup::" + if command -v git &> /dev/null; then + git config --system --add safe.directory "${{ github.workspace }}" + fi + GITHUB_TOKEN="${{ inputs.token }}" if [ -n "${GITHUB_TOKEN}" ]; then git config --system url."https://api:${GITHUB_TOKEN}@github.com/".insteadOf "https://github.com/" @@ -155,7 +181,7 @@ runs: echo "::endgroup::" - name: Build packages - uses: kohlerdominik/docker-run-action@v1.2.0 + uses: kohlerdominik/docker-run-action@v2.0.0 with: image: ${{ inputs.docker-image }} options: --cap-add=NET_ADMIN From bd363d7a067217b570f50495938e7722ce85b6ad Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marco=20Trevisan=20=28Trevi=C3=B1o=29?= Date: Mon, 26 Feb 2024 03:51:05 +0100 Subject: [PATCH 15/23] build-debian: Use more featured version number if in a git repo In case a git repository is setup, then use version number that includes the revision list and use actual author name --- gh-actions/common/build-debian/action.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/gh-actions/common/build-debian/action.yml b/gh-actions/common/build-debian/action.yml index 754dcbb1..53aafd3c 100644 --- a/gh-actions/common/build-debian/action.yml +++ b/gh-actions/common/build-debian/action.yml @@ -63,6 +63,11 @@ runs: echo DEBFULLNAME="GitHub actions runner" >> $GITHUB_ENV echo DEBEMAIL="noreply@github.com" >> $GITHUB_ENV + + if git status --porcelain &>/dev/null; then + echo DEBFULLNAME="$(git log -1 --format='%an' HEAD) - GH Action" >> $GITHUB_ENV + echo DEBEMAIL="$(git log -1 --format='%ae' HEAD)" >> $GITHUB_ENV + fi echo "::endgroup::" - name: Prepare source package From f19243da3209ffbd9028552cdb292d9b7ef4401c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marco=20Trevisan=20=28Trevi=C3=B1o=29?= Date: Mon, 26 Feb 2024 04:15:00 +0100 Subject: [PATCH 16/23] build-debian: Remove trailing whitespace --- gh-actions/common/build-debian/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gh-actions/common/build-debian/action.yml b/gh-actions/common/build-debian/action.yml index 53aafd3c..2db897e7 100644 --- a/gh-actions/common/build-debian/action.yml +++ b/gh-actions/common/build-debian/action.yml @@ -175,7 +175,7 @@ runs: BUILD_INPUT_DIR="${BUILD_INPUT_BASEDIR}/source" echo BUILD_INPUT_DIR="${BUILD_INPUT_DIR}" >> $GITHUB_ENV echo "::endgroup::" - + echo "::group::Create build output directory" echo BUILD_OUTPUT_DIR="$( mktemp --directory --tmpdir="${PWD}" )" >> $GITHUB_ENV echo "::endgroup::" From 39b462ff0b7ce5877a0f0d0bd327068c392150f7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marco=20Trevisan=20=28Trevi=C3=B1o=29?= Date: Mon, 26 Feb 2024 04:22:45 +0100 Subject: [PATCH 17/23] build-debian: Show binaries contents and info after building It can provide some quick information about what's in the packages --- gh-actions/common/build-debian/action.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/gh-actions/common/build-debian/action.yml b/gh-actions/common/build-debian/action.yml index 2db897e7..fcbcac54 100644 --- a/gh-actions/common/build-debian/action.yml +++ b/gh-actions/common/build-debian/action.yml @@ -235,6 +235,14 @@ runs: mv -v ../*"_${{ env.PKG_VERSION }}_"*.deb "${{ env.BUILD_OUTPUT_DIR }}" + echo "::group::Show binaries information" + for i in "${{ env.BUILD_OUTPUT_DIR }}"/*.deb; do + echo "$(basename "$i")" + dpkg --info "$i" + dpkg --contents "$i" + done + echo "::endgroup::" + - name: Upload artifacts uses: actions/upload-artifact@v4 with: From 2c964f127765e4ba8cb5a17088fd477d1f54eb65 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marco=20Trevisan=20=28Trevi=C3=B1o=29?= Date: Mon, 26 Feb 2024 05:21:42 +0100 Subject: [PATCH 18/23] build-debian: Add IDs to upload steps so that can be referenced as outputs --- .github/workflows/test-build-deb.yaml | 2 ++ gh-actions/common/build-debian/action.yml | 2 ++ 2 files changed, 4 insertions(+) diff --git a/.github/workflows/test-build-deb.yaml b/.github/workflows/test-build-deb.yaml index 817f5b0c..f58b6aa6 100644 --- a/.github/workflows/test-build-deb.yaml +++ b/.github/workflows/test-build-deb.yaml @@ -60,6 +60,8 @@ jobs: outputs: pkg-name: ${{ env.PKG_NAME }} pkg-version: ${{ env.PKG_VERSION }} + source-pkg: ${{ steps.build-debian-source-package-upload-step.outputs.artifact-url }} + binaries: ${{ steps.build-debian-binary-packages-upload-step.outputs.artifact-url }} steps: - name: Install dependencies diff --git a/gh-actions/common/build-debian/action.yml b/gh-actions/common/build-debian/action.yml index fcbcac54..30f72a85 100644 --- a/gh-actions/common/build-debian/action.yml +++ b/gh-actions/common/build-debian/action.yml @@ -158,6 +158,7 @@ runs: - name: Uploading source packages uses: actions/upload-artifact@v4 + id: build-debian-source-package-upload-step with: name: ${{ env.PKG_NAME }}_${{ env.PKG_VERSION }}-debian-source path: ${{ env.SOURCE_OUTPUT_DIR }}/ @@ -244,6 +245,7 @@ runs: echo "::endgroup::" - name: Upload artifacts + id: build-debian-binary-packages-upload-step uses: actions/upload-artifact@v4 with: name: ${{ env.PKG_NAME }}_${{ env.PKG_VERSION }}-debian-packages From f3c75c6b216b251e0b7cdea6b50d7c34516110d1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marco=20Trevisan=20=28Trevi=C3=B1o=29?= Date: Mon, 26 Feb 2024 14:01:41 +0100 Subject: [PATCH 19/23] build-debian: Ensure we show group message as first thing So that github groups things better --- gh-actions/common/build-debian/action.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gh-actions/common/build-debian/action.yml b/gh-actions/common/build-debian/action.yml index 30f72a85..73e97764 100644 --- a/gh-actions/common/build-debian/action.yml +++ b/gh-actions/common/build-debian/action.yml @@ -53,9 +53,9 @@ runs: - name: Set up source package build shell: bash run: | + echo "::group::Create local version with commit and docker container" set -eu - echo "::group::Create local version with commit and docker container" cd '${{ inputs.source-dir }}' # Short commit to avoid "package-has-long-file-name" @@ -167,9 +167,9 @@ runs: - name: Set up package build shell: bash run: | + echo "::group::Create build input directory" set -eu - echo "::group::Create build input directory" # Appending /source because 'dpkg-source --extract' needs the output directory to be non-existent BUILD_INPUT_BASEDIR="$( mktemp --directory --tmpdir="${PWD}" )" echo BUILD_INPUT_BASEDIR="${BUILD_INPUT_BASEDIR}" >> $GITHUB_ENV From f17b5d99796113dcc8caf66f680d4e53406676ca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marco=20Trevisan=20=28Trevi=C3=B1o=29?= Date: Mon, 26 Feb 2024 14:23:04 +0100 Subject: [PATCH 20/23] Revert "build-debian: Add DEB_BUILD_OPTIONS input" This reverts commit 3f553109f5f838a93e26c517a6839d358281e1ce. --- gh-actions/common/build-debian/action.yml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/gh-actions/common/build-debian/action.yml b/gh-actions/common/build-debian/action.yml index 73e97764..7b379760 100644 --- a/gh-actions/common/build-debian/action.yml +++ b/gh-actions/common/build-debian/action.yml @@ -17,10 +17,6 @@ inputs: required: false # FIXME: this should default to '', but we don't want to break job depending on us for now default: 'ca-certificates git' - DEB_BUILD_OPTIONS: - required: false - default: '' - description: The DEB_BUILD_OPTIONS environment variable that is used during builds. # The process: @@ -119,7 +115,6 @@ runs: image: ${{ inputs.docker-image }} environment: | DEBIAN_FRONTEND=noninteractive - DEB_BUILD_OPTIONS=${{ inputs.DEB_BUILD_OPTIONS }} volumes: ${{ github.workspace }}:${{ github.workspace }} workdir: ${{ github.workspace }}/${{ inputs.source-dir }} shell: bash @@ -193,7 +188,6 @@ runs: options: --cap-add=NET_ADMIN environment: | DEBIAN_FRONTEND=noninteractive - DEB_BUILD_OPTIONS=${{ inputs.DEB_BUILD_OPTIONS }} workdir: ${{ env.BUILD_INPUT_DIR }} volumes: | ${{ env.BUILD_INPUT_BASEDIR }}:${{ env.BUILD_INPUT_BASEDIR }} From 33369550a0607b50bf7037741c32124b953aee5a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marco=20Trevisan=20=28Trevi=C3=B1o=29?= Date: Mon, 26 Feb 2024 15:51:14 +0100 Subject: [PATCH 21/23] build-debian: Include distribution in changelog release --- gh-actions/common/build-debian/action.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/gh-actions/common/build-debian/action.yml b/gh-actions/common/build-debian/action.yml index 7b379760..6fac9710 100644 --- a/gh-actions/common/build-debian/action.yml +++ b/gh-actions/common/build-debian/action.yml @@ -88,7 +88,8 @@ runs: echo "::group::Update debian package changelog" dch --local "~$(lsb_release -r -s)+git${{ env.VERSION_REF }}" \ - "Github build. Run id: ${{ github.run_id }}. Run number: ${{ github.run_number }}." + "Github build. Run id: ${{ github.run_id }}. Run number: ${{ github.run_number }}." \ + --distribution "$(lsb_release -c -s)" dpkg-parsechangelog echo "::endgroup::" From 005736acf129663126f2d9d1d36c90aaa462b625 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marco=20Trevisan=20=28Trevi=C3=B1o=29?= Date: Mon, 26 Feb 2024 15:52:58 +0100 Subject: [PATCH 22/23] debian-build: Use version number so that it can override last release Use a +git syntax and there we use incremental version values. Finally add the distro version as a minor factor, so that builds on the same code for different distros only rely on that --- gh-actions/common/build-debian/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gh-actions/common/build-debian/action.yml b/gh-actions/common/build-debian/action.yml index 6fac9710..33d84058 100644 --- a/gh-actions/common/build-debian/action.yml +++ b/gh-actions/common/build-debian/action.yml @@ -87,7 +87,7 @@ runs: echo "::endgroup::" echo "::group::Update debian package changelog" - dch --local "~$(lsb_release -r -s)+git${{ env.VERSION_REF }}" \ + dch --local "+git${{ env.VERSION_REF }}~$(lsb_release -r -s)." \ "Github build. Run id: ${{ github.run_id }}. Run number: ${{ github.run_number }}." \ --distribution "$(lsb_release -c -s)" From 542f160f29cb86d1ffd7b0de57b63304ab4ea6dd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marco=20Trevisan=20=28Trevi=C3=B1o=29?= Date: Tue, 12 Nov 2024 23:17:04 +0100 Subject: [PATCH 23/23] build-debian: Install fakeroot by default Since we're building as user --- gh-actions/common/build-debian/action.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/gh-actions/common/build-debian/action.yml b/gh-actions/common/build-debian/action.yml index 33d84058..a389eb12 100644 --- a/gh-actions/common/build-debian/action.yml +++ b/gh-actions/common/build-debian/action.yml @@ -221,6 +221,10 @@ runs: apt autoremove echo "::endgroup::" + echo "::group::Install fakeroot" + apt install fakeroot + echo "::endgroup::" + echo "::group::Install build dependencies" apt build-dep . echo "::endgroup::"