diff --git a/.github/actions/python-prepare/action.yaml b/.github/actions/python-prepare/action.yaml index 00224f21..6dd49f9f 100644 --- a/.github/actions/python-prepare/action.yaml +++ b/.github/actions/python-prepare/action.yaml @@ -22,20 +22,42 @@ inputs: Path to the pyproject.toml file, relative to the repository root. Defaults to `pyproject.toml`. +outputs: + pip-extra-index-url: + description: | + The PIP_EXTRA_INDEX_URL value used to reach AWS CodeArtifact for this run + (empty string if no code-artefact-role was supplied). Requirements files + generated or installed by this action never have the index URL embedded + in them, so later steps needing CodeArtifact access (e.g. `python -m build`) + must set this as their own PIP_EXTRA_INDEX_URL env var. + value: ${{ steps.code-artifact-login.outputs.token != '' && format('https://aws:{0}@telicent-{1}.d.codeartifact.eu-west-2.amazonaws.com/pypi/telicent-code-artifacts/simple/', steps.code-artifact-login.outputs.token, steps.creds.outputs.aws-account-id) || '' }} + runs: using: composite steps: - name: Checkout uses: actions/checkout@v6 + - name: Set up Python ${{ inputs.version }} uses: actions/setup-python@v5.6.0 with: python-version: ${{ inputs.version }} - - name: Prepare pip and install pip-tools - shell: ${{ runner.os == 'Windows' && 'pwsh' || 'bash' }} + + - name: Resolve requirements path + shell: bash run: | - python -m pip install --upgrade "pip<25.3" - python -m pip install pip-tools + req_dir=$(dirname "${{ inputs.pyproject-toml-path }}") + echo "REQUIREMENTS_PATH=$req_dir/requirements.txt" >> "$GITHUB_ENV" + + - name: Prepare pip + shell: ${{ runner.os == 'Windows' && 'pwsh' || 'bash' }} + run: python -m pip install --upgrade "pip<25.3" + + - name: Install pip-tools + if: hashFiles(env.REQUIREMENTS_PATH) == '' + shell: ${{ runner.os == 'Windows' && 'pwsh' || 'bash' }} + run: python -m pip install pip-tools + - name: Configure CodeArtifact AWS credentials if: inputs.code-artefact-role != '' id: creds @@ -44,6 +66,7 @@ runs: role-to-assume: ${{ inputs.code-artefact-role }} aws-region: eu-west-2 mask-aws-account-id: true + - name: CodeArtifact login id: code-artifact-login if: inputs.code-artefact-role != '' @@ -51,12 +74,31 @@ runs: with: domain: telicent owner: "${{ steps.creds.outputs.aws-account-id }}" + + - name: Warn if no committed requirements.txt was found + if: hashFiles(env.REQUIREMENTS_PATH) == '' + shell: bash + run: echo "::warning::No committed $REQUIREMENTS_PATH found - generating one from ${{ inputs.pyproject-toml-path }} on the fly. This project is out of date; commit a hash-pinned requirements.txt (pip-compile --generate-hashes) instead." + - name: Generate requirements + if: hashFiles(env.REQUIREMENTS_PATH) == '' shell: ${{ runner.os == 'Windows' && 'pwsh' || 'bash' }} - run: python -m piptools compile ${{ inputs.extra-args }} -o requirements.txt ${{ inputs.pyproject-toml-path }} + run: python -m piptools compile --generate-hashes --no-emit-index-url ${{ inputs.extra-args }} -o ${{ env.REQUIREMENTS_PATH }} ${{ inputs.pyproject-toml-path }} env: PIP_EXTRA_INDEX_URL: ${{ steps.code-artifact-login.outputs.token != '' && format('https://aws:{0}@telicent-{1}.d.codeartifact.eu-west-2.amazonaws.com/pypi/telicent-code-artifacts/simple/', steps.code-artifact-login.outputs.token, steps.creds.outputs.aws-account-id) || '' }} + + - name: Verify requirements are hash-pinned + if: hashFiles(env.REQUIREMENTS_PATH) != '' + shell: bash + run: | + if ! grep -q -- '--hash=' "$REQUIREMENTS_PATH"; then + echo "::error::$REQUIREMENTS_PATH exists but contains no --hash= entries. Committed requirements files must be generated with 'pip-compile --generate-hashes' for supply-chain integrity." + exit 1 + fi + - name: Install requirements if: inputs.install-requirements == 'true' shell: ${{ runner.os == 'Windows' && 'pwsh' || 'bash' }} - run: pip install -r requirements.txt + run: pip install --require-hashes -r ${{ env.REQUIREMENTS_PATH }} + env: + PIP_EXTRA_INDEX_URL: ${{ steps.code-artifact-login.outputs.token != '' && format('https://aws:{0}@telicent-{1}.d.codeartifact.eu-west-2.amazonaws.com/pypi/telicent-code-artifacts/simple/', steps.code-artifact-login.outputs.token, steps.creds.outputs.aws-account-id) || '' }} diff --git a/.github/actions/python-prepare/test-fixtures/generate-only/pyproject.toml b/.github/actions/python-prepare/test-fixtures/generate-only/pyproject.toml new file mode 100644 index 00000000..b536a7a0 --- /dev/null +++ b/.github/actions/python-prepare/test-fixtures/generate-only/pyproject.toml @@ -0,0 +1,8 @@ +[project] +name = "test-fixture-generate-only" +version = "0.0.1" +dependencies = ["certifi==2024.8.30"] + +[build-system] +requires = ["setuptools"] +build-backend = "setuptools.build_meta" diff --git a/.github/actions/python-prepare/test-fixtures/missing-hashes/pyproject.toml b/.github/actions/python-prepare/test-fixtures/missing-hashes/pyproject.toml new file mode 100644 index 00000000..48dd02fe --- /dev/null +++ b/.github/actions/python-prepare/test-fixtures/missing-hashes/pyproject.toml @@ -0,0 +1,8 @@ +[project] +name = "test-fixture-missing-hashes" +version = "0.0.1" +dependencies = ["certifi==2024.8.30"] + +[build-system] +requires = ["setuptools"] +build-backend = "setuptools.build_meta" diff --git a/.github/actions/python-prepare/test-fixtures/missing-hashes/requirements.txt b/.github/actions/python-prepare/test-fixtures/missing-hashes/requirements.txt new file mode 100644 index 00000000..8cbfdc93 --- /dev/null +++ b/.github/actions/python-prepare/test-fixtures/missing-hashes/requirements.txt @@ -0,0 +1,6 @@ +# +# This requirements.txt was committed without hashes (e.g. hand-written or +# generated with plain `pip-compile`, not `pip-compile --generate-hashes`). +# It is used as a fixture to verify python-prepare rejects it. +# +certifi==2024.8.30 diff --git a/.github/actions/python-prepare/test-fixtures/valid-hashes/pyproject.toml b/.github/actions/python-prepare/test-fixtures/valid-hashes/pyproject.toml new file mode 100644 index 00000000..9f500dca --- /dev/null +++ b/.github/actions/python-prepare/test-fixtures/valid-hashes/pyproject.toml @@ -0,0 +1,8 @@ +[project] +name = "test-fixture-valid-hashes" +version = "0.0.1" +dependencies = ["certifi==2024.8.30"] + +[build-system] +requires = ["setuptools"] +build-backend = "setuptools.build_meta" diff --git a/.github/actions/python-prepare/test-fixtures/valid-hashes/requirements.txt b/.github/actions/python-prepare/test-fixtures/valid-hashes/requirements.txt new file mode 100644 index 00000000..5aaef15e --- /dev/null +++ b/.github/actions/python-prepare/test-fixtures/valid-hashes/requirements.txt @@ -0,0 +1,10 @@ +# +# This file is autogenerated by pip-compile with Python 3.12 +# by the following command: +# +# pip-compile --generate-hashes --no-emit-index-url --output-file=requirements.txt pyproject.toml +# +certifi==2024.8.30 \ + --hash=sha256:922820b53db7a7257ffbda3f597266d435245903d80737e34f8a45ff3e3230d8 \ + --hash=sha256:bec941d2aa8195e248a60b31ff9f0558284cf01a52591ceda73ea9afffd69fd9 + # via test-fixture-valid-hashes (pyproject.toml) diff --git a/.github/actions/python-test/action.yaml b/.github/actions/python-test/action.yaml index e79cf93b..dc3e973d 100644 --- a/.github/actions/python-test/action.yaml +++ b/.github/actions/python-test/action.yaml @@ -35,6 +35,7 @@ runs: using: composite steps: - name: Prepare Python + id: prepare uses: telicent-oss/shared-workflows/.github/actions/python-prepare@main with: code-artefact-role: ${{ inputs.code-artefact-role}} @@ -51,7 +52,13 @@ runs: - name: Unit tests shell: ${{ runner.os == 'Windows' && 'pwsh' || 'bash' }} run: ${{ inputs.test-command }} + - name: Install build + if: ${{ inputs.skip-build == 'false' }} + shell: ${{ runner.os == 'Windows' && 'pwsh' || 'bash' }} + run: python -m pip install build - name: Build if: ${{ inputs.skip-build == 'false' }} shell: ${{ runner.os == 'Windows' && 'pwsh' || 'bash' }} - run: python -m build \ No newline at end of file + run: python -m build + env: + PIP_EXTRA_INDEX_URL: ${{ steps.prepare.outputs.pip-extra-index-url }} \ No newline at end of file diff --git a/.github/workflows/test-changes-to-python-prepare-action.yaml b/.github/workflows/test-changes-to-python-prepare-action.yaml new file mode 100644 index 00000000..d59ee6e9 --- /dev/null +++ b/.github/workflows/test-changes-to-python-prepare-action.yaml @@ -0,0 +1,77 @@ +name: Test Changes to Python Prepare Action + +on: + push: + paths: + - .github/actions/python-prepare/** + - .github/workflows/test-changes-to-python-prepare-action.yaml + workflow_dispatch: + +jobs: + no-requirements-file-generates-hashed-one: + strategy: + matrix: + os: [ ubuntu-latest, windows-latest, macos-latest ] + fail-fast: false + runs-on: ${{ matrix.os }} + permissions: + contents: read + steps: + - uses: actions/checkout@v6 + - name: Prepare Python + uses: ./.github/actions/python-prepare + with: + version: "3.12" + install-requirements: "true" + pyproject-toml-path: .github/actions/python-prepare/test-fixtures/generate-only/pyproject.toml + - name: "Fail if requirements.txt was not generated" + shell: bash + run: test -f .github/actions/python-prepare/test-fixtures/generate-only/requirements.txt + - name: "Fail if generated requirements.txt has no hashes" + shell: bash + run: grep -q -- '--hash=' .github/actions/python-prepare/test-fixtures/generate-only/requirements.txt + + committed-hashed-file-is-used-as-is: + strategy: + matrix: + os: [ ubuntu-latest, windows-latest, macos-latest ] + fail-fast: false + runs-on: ${{ matrix.os }} + permissions: + contents: read + steps: + - uses: actions/checkout@v6 + - name: Prepare Python + uses: ./.github/actions/python-prepare + with: + version: "3.12" + install-requirements: "true" + pyproject-toml-path: .github/actions/python-prepare/test-fixtures/valid-hashes/pyproject.toml + - name: "Fail if the committed requirements.txt was regenerated/modified" + shell: bash + run: git diff --exit-code -- .github/actions/python-prepare/test-fixtures/valid-hashes/requirements.txt + + committed-unhashed-file-is-rejected: + strategy: + matrix: + os: [ ubuntu-latest, windows-latest, macos-latest ] + fail-fast: false + runs-on: ${{ matrix.os }} + permissions: + contents: read + steps: + - uses: actions/checkout@v6 + - name: "Prepare Python (expected to fail)" + id: prepare + continue-on-error: true + uses: ./.github/actions/python-prepare + with: + version: "3.12" + install-requirements: "true" + pyproject-toml-path: .github/actions/python-prepare/test-fixtures/missing-hashes/pyproject.toml + - name: "Fail the job if python-prepare did not reject the un-hashed file" + if: steps.prepare.outcome != 'failure' + shell: bash + run: | + echo "Expected python-prepare to fail on a committed requirements.txt with no --hash= entries, but it succeeded" + exit 1