diff --git a/.github/workflows/branch.yml b/.github/workflows/branch.yml index e6fc452..34733ad 100644 --- a/.github/workflows/branch.yml +++ b/.github/workflows/branch.yml @@ -54,24 +54,22 @@ jobs: python_versions=$(curl --fail --silent --show-error \ https://raw.githubusercontent.com/actions/python-versions/main/versions-manifest.json) - os_version=$(. /etc/os-release; echo "$VERSION_ID") get_version() { local version version=$(echo "$python_versions" \ - | jq -r --arg series "$1" --arg os_version "$os_version" ' + | jq -r --arg series "$1" ' .[] | select(.stable == true) | select(.version | startswith($series + ".")) | select(any(.files[]; - .platform == "linux" and .platform_version == $os_version - and .arch == "x64")) + .platform == "linux" and .arch == "x64")) | .version ' \ | sort -V \ | tail -1) || return 1 if [[ -z "$version" ]]; then - echo "No stable Python $1 x64 build available for Ubuntu $os_version" >&2 + echo "No stable Python $1 x64 build available" >&2 return 1 fi echo "$version" @@ -112,7 +110,7 @@ jobs: cat $GITHUB_OUTPUT Collect: - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 needs: [ Setup ] permissions: @@ -219,7 +217,7 @@ jobs: if-no-files-found: error SecurityScan: - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 needs: [ Setup, Collect ] permissions: @@ -524,16 +522,19 @@ jobs: - name: Copy Markdowns to ISO run: cp -v *.md ISO/ + - name: Copy scripts to ISO + run: cp -v *.sh ISO/ + + - name: Copy package manifests to ISO + run: | + cp -v requirements*.txt ISO/ + - name: Move evidence to ISO/evidence run: | mkdir -p ISO/evidence mv -v ISO/*report* ISO/evidence/ mv -v ISO/*sbom* ISO/evidence/ - - name: Copy package manifests to ISO - run: | - cp -v requirements*.txt ISO/ - - name: Package source repository into ISO run: | echo -e "\nPackaging source repository" diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index be5a847..31a97ea 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -54,24 +54,22 @@ jobs: python_versions=$(curl --fail --silent --show-error \ https://raw.githubusercontent.com/actions/python-versions/main/versions-manifest.json) - os_version=$(. /etc/os-release; echo "$VERSION_ID") get_version() { local version version=$(echo "$python_versions" \ - | jq -r --arg series "$1" --arg os_version "$os_version" ' + | jq -r --arg series "$1" ' .[] | select(.stable == true) | select(.version | startswith($series + ".")) | select(any(.files[]; - .platform == "linux" and .platform_version == $os_version - and .arch == "x64")) + .platform == "linux" and .arch == "x64")) | .version ' \ | sort -V \ | tail -1) || return 1 if [[ -z "$version" ]]; then - echo "No stable Python $1 x64 build available for Ubuntu $os_version" >&2 + echo "No stable Python $1 x64 build available" >&2 return 1 fi echo "$version" @@ -112,7 +110,7 @@ jobs: cat $GITHUB_OUTPUT Collect: - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 needs: [ Setup ] permissions: @@ -219,7 +217,7 @@ jobs: if-no-files-found: error SecurityScan: - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 needs: [ Setup, Collect ] permissions: diff --git a/.gitignore b/.gitignore index 637ee79..4b29572 100644 --- a/.gitignore +++ b/.gitignore @@ -8,9 +8,11 @@ dist/ pypi*/ pypi coverage/ +__pycache__/ docs/ # Ignore Files +*.pyc *.tgz *.tar.gz *.gz diff --git a/README.md b/README.md index a35a22b..493cbcf 100644 --- a/README.md +++ b/README.md @@ -21,9 +21,11 @@ Use `upload.sh` on the highside to setup the venv to enable uploading ## Python runtime versions -Collection jobs select the latest stable x64 Python build for the runner's Ubuntu -version from the `actions/python-versions` manifest used by `setup-python`, rather -than assuming every python.org release has an available Actions build. +Collection jobs select the latest stable Linux x64 Python build from the +`actions/python-versions` manifest used by `setup-python`, rather than assuming +every python.org release has an available Actions build. Collection and security +scan jobs use Ubuntu 24.04 so Python 3.9 remains available; moving these jobs to a +newer runner requires checking build availability for every collected series. Both collection workflows read `python-runtimes.json`. Its `source_versions` list selects the Python source archives; `windows_versions` independently selects diff --git a/Vagrantfile b/Vagrantfile new file mode 100644 index 0000000..b0145c8 --- /dev/null +++ b/Vagrantfile @@ -0,0 +1,125 @@ +# -*- mode: ruby -*- +# vi: set ft=ruby : + +# All Vagrant configuration is done below. +Vagrant.configure("2") do |config| + # The most common configuration options are documented and commented below. + # For a complete reference, please see the online documentation at + # https://docs.vagrantup.com. + + # Every Vagrant development environment requires a box. You can search for + # boxes at https://vagrantcloud.com/search. + config.vm.box = "oraclelinux/9-btrfs" + config.vm.box_url = "https://oracle.github.io/vagrant-projects/boxes/oraclelinux/9-btrfs.json" + config.vm.hostname = "Node-Collector" + + config.ssh.key_type = :ecdsa521 # Requires Vagrant 2.4.1 + + ############################################################################ + # Provider-specific configuration # + ############################################################################ + config.vm.provider "virtualbox" do |vb| + # Set Name + vb.name = "Node Collector - OEL9" + + # Display the VirtualBox GUI when booting the machine + vb.gui = false + + # Set up VM options + vb.customize ["modifyvm", :id, "--vm-process-priority", "normal"] + vb.customize ["modifyvm", :id, "--graphiccontroller", "vmsvga"] + vb.customize ["modifyvm", :id, "--vram", "160"] + vb.customize ["modifyvm", :id, "--clipboard-mode", "bidirectional"] + vb.customize ["modifyvm", :id, "--draganddrop", "bidirectional"] + vb.customize ["modifyvm", :id, "--usbxhci", "on"] + vb.customize ["modifyvm", :id, "--audioin", "on"] + vb.customize ["modifyvm", :id, "--audiocontroller", "hda"] + vb.customize ["modifyvm", :id, "--vrde", "off"] + end + + ############################################################################ + # File copy provisioners # + ############################################################################ + config.vm.provision "file", source: "~/.ssh", destination: ".ssh" + config.vm.provision "file", source: "~/.gitconfig", destination: ".gitconfig" + + ############################################################################ + # Shell script provisioner # + ############################################################################ + config.vm.provision "shell", inline: <<-'SHELL' + ############################################################################ + # Add Software # + ############################################################################ + # Add EPEL + echo -e "\nInstalling additional repos\n" + dnf install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-9.noarch.rpm + curl -fsSL https://cli.github.com/packages/rpm/gh-cli.repo >\ + "/etc/yum.repos.d/github-cli.repo" + echo + /usr/bin/crb enable + + # Software Update + echo -e "\nInstalling Updates\n" + dnf update -y + + # Install Container/Additional Software + echo -e "\nInstalling additional software\n" + dnf install -y podman skopeo podman-docker tmux tree \ + git git-lfs gh rsync mkisofs isomd5sum \ + python3.12 python3.12-venv python3.12-pip + + # Setup Rootless Podman + echo -e "\nSetting up rootless podman" + sysctl user.max_user_namespaces=15000 + sed -i 's/user.max_user_namespaces=0/user.max_user_namespaces=15000/i' /etc/sysctl.conf + usermod --add-subuids 200000-201000 --add-subgids 200000-201000 vagrant + groupadd -r docker + usermod -aG docker vagrant + touch /etc/containers/nodocker + + # Import .ssh to vagrant user + echo -e "\nSetting up vagrant user .ssh" + cp -v /root/.gitconfig /home/vagrant/ + git config --global http.sslBackend openssl + cat /home/vagrant/.ssh/id*.pub >> /home/vagrant/.ssh/authorized_keys + cat /root/.ssh/id*.pub >> /root/.ssh/authorized_keys + chown -R vagrant:vagrant /home/vagrant + chmod -R 600 /home/vagrant/.ssh + chmod 700 /home/vagrant/.ssh + + echo -e "\nDone Updating. Rebooting.\n" + SHELL + + # Reboot to enable updates + config.vm.provision 'shell', reboot: true + + # Enable FIPS + config.vm.provision "shell", inline: <<-'SHELL' + echo -e "\nEnabling FIPS\n" + fips-mode-setup --enable + SHELL + + # Reboot to enable FIPS + config.vm.provision 'shell', reboot: true + + # Run Local routine to set up environment + config.vm.provision "shell", privileged: false, + env: { + GITHUB_TOKEN:ENV['GITHUB_TOKEN'] + }, inline: <<-'SHELL' + echo -e "\nRunning user provisioning\n" + if [[ -z "$GITHUB_TOKEN" ]]; then + echo "GITHUB_TOKEN undefined. Will Prompt authentication via browser" + gh auth login + else + echo "GITHUB_TOKEN detected. Respecting token." + fi + export DATE=`date '+%Y%m%d-%H%M'` + + ./setup.sh + ./collect.sh + # ./mkiso.sh + + SHELL + +end diff --git a/kickstart.sh b/kickstart.sh index 5b29c81..c206cb3 100755 --- a/kickstart.sh +++ b/kickstart.sh @@ -3,23 +3,54 @@ set -euo pipefail PYTHON="${PYTHON:-python3.12}" VENV="${VENV:-.venv}" -WHEELHOUSE="${WHEELHOUSE:-pypi}" -if [[ ${LOCK+x} ]]; then - echo "LOCK is set." - REQFILE="${REQFILE:-requirements-lock.txt}" -else - REQFILE="${REQFILE:-requirements.txt}" -fi +command -v "${PYTHON}" >/dev/null 2>&1 || { + echo "ERROR: Python interpreter not found: ${PYTHON}" >&2 + exit 1 +} -CA_BUNDLE="${CA_BUNDLE:-/etc/ssl/certs/ca-certificates.crt}" -export TWINE_CA_BUNDLE="${CA_BUNDLE}" +if [[ -z "${WHEELHOUSE:-}" ]]; then + PYTHON_SERIES=$( + "${PYTHON}" -c \ + 'import sys; print(f"{sys.version_info.major}.{sys.version_info.minor}")' + ) -command -v "${PYTHON}" >/dev/null 2>&1 || { - echo "ERROR: ${PYTHON} is not installed." >&2 + shopt -s nullglob + matches=( "pypi-${PYTHON_SERIES}."* ) + shopt -u nullglob + + case "${#matches[@]}" in + 0) + echo "ERROR: No pypi-${PYTHON_SERIES}.* directory found." >&2 + exit 1 + ;; + 1) + WHEELHOUSE="${matches[0]}" + ;; + *) + echo "ERROR: Multiple matching wheelhouses found:" >&2 + printf ' %s\n' "${matches[@]}" >&2 + echo "Set WHEELHOUSE explicitly to select one." >&2 + exit 1 + ;; + esac +fi + +[[ -d "${WHEELHOUSE}" ]] || { + echo "ERROR: Wheelhouse is not a directory: ${WHEELHOUSE}" >&2 exit 1 } +echo "Using Python: $("${PYTHON}" --version 2>&1)" +echo "Using wheelhouse: ${WHEELHOUSE}" + +if [[ ${LOCK+x} ]]; then + echo "LOCK is set." + REQFILE="${REQFILE:-requirements-lock.txt}" +else + REQFILE="${REQFILE:-requirements-dev.txt}" +fi + [[ -d "${WHEELHOUSE}" ]] || { echo "ERROR: Wheelhouse not found: ${WHEELHOUSE}" >&2 exit 1 @@ -41,13 +72,13 @@ echo "Python environment:" echo echo "Installing packages from ${WHEELHOUSE}..." if [[ ${LOCK+x} ]]; then - "${VENV}/bin/python" -m pip install \ + "${VENV}/bin/python" -m pip install \ --no-index \ --find-links "${WHEELHOUSE}" \ --require-hashes \ --requirement "${REQFILE}" else - "${VENV}/bin/python" -m pip install \ + "${VENV}/bin/python" -m pip install \ --no-index \ --find-links "${WHEELHOUSE}" \ --requirement "${REQFILE}" diff --git a/mkiso.sh b/mkiso.sh new file mode 100755 index 0000000..7a61606 --- /dev/null +++ b/mkiso.sh @@ -0,0 +1,41 @@ +#!/bin/bash +set -e +if [ $1 ]; then + ROOTDIR=$1 + echo -e "\n$ROOTDIR" +else + echo -e "\nUsage Instructions:" + echo -e "\t $0 directory" + echo -e "\nUsing default + ROOTDIR=${basename $PWD} + echo -e "\n$ROOTDIR" +fi + +if [ -n $DATE ]; then + DATE=`date '+%Y%m%d-%H%M'` +fi + +echo "Run Collection Script for $ROOTDIR.txt" +./collect.sh $ROOTDIR.txt + +echo "Copy README to ISO" +cp -v README.md $ROOTDIR/ + +echo "Make ISO" +echo -e "\nMaking ISO of $1" +mkisofs -J -R -v -T -l \ + -V $ROOTDIR-$DATE \ + -A $ROOTDIR-$DATE \ + -o $ROOTDIR-$DATE.iso \ + $ROOTDIR + +echo "Implant MD5 sum into iso" +implantisomd5 $ROOTDIR-$DATE.iso + +echo "Generate Hash" +sha256sum -b $ROOTDIR-$DATE.iso >>\ + $ROOTDIR-$DATE.iso.sha +cat $ROOTDIR-$DATE.iso.sha + +echo "Show ISO & Hash" +ls -Alht *.iso* \ No newline at end of file diff --git a/requirements-dev.txt b/requirements-dev.txt index ce39f9c..6987758 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -13,4 +13,5 @@ reprotest twine virtualenv wheel -ruff \ No newline at end of file +ruff +gcovr diff --git a/tests/test_collection_versions.py b/tests/test_collection_versions.py index 5892fe1..71e12d1 100644 --- a/tests/test_collection_versions.py +++ b/tests/test_collection_versions.py @@ -13,14 +13,11 @@ @unittest.skipUnless( - shutil.which("bash") and shutil.which("jq") and Path("/etc/os-release").exists(), - "Collection version discovery requires bash, jq, and /etc/os-release", + shutil.which("bash") and shutil.which("jq"), + "Collection version discovery requires bash and jq", ) class CollectionVersionTests(unittest.TestCase): def setUp(self): - self.os_version = subprocess.check_output( - ["bash", "-c", '. /etc/os-release; echo "$VERSION_ID"'], text=True - ).strip() self.manifest = [ self.release("3.9.25"), self.release("3.11.14"), @@ -35,7 +32,7 @@ def release(self, version, stable=True, **file_overrides): "stable": stable, "files": [{ "platform": "linux", - "platform_version": self.os_version, + "platform_version": "24.04", "arch": "x64", **file_overrides, }], @@ -67,9 +64,9 @@ def discover(self, workflow, manifest=None, curl_status=0): ) return result, output.read_text() if output.exists() else "" - def test_selects_latest_supported_stable_build_for_each_series(self): + def test_selects_latest_stable_build_independent_of_ubuntu_version(self): self.manifest.extend([ - self.release("3.13.17", platform_version="unsupported"), + self.release("3.13.17", platform_version="22.04"), self.release("3.13.18", platform="darwin"), self.release("3.13.19", arch="arm64"), self.release("3.13.20", arch="x64-freethreaded"), @@ -81,9 +78,19 @@ def test_selects_latest_supported_stable_build_for_each_series(self): result, output = self.discover(workflow) self.assertEqual(result.returncode, 0, result.stderr) self.assertEqual(output.splitlines(), [ - "v309=3.9.25", "v311=3.11.14", "v312=3.12.12", "v313=3.13.10", + "v309=3.9.25", "v311=3.11.14", "v312=3.12.12", "v313=3.13.17", ]) + def test_python_matrix_jobs_use_runner_with_python_39_builds(self): + for workflow in WORKFLOWS: + contents = (ROOT / ".github" / "workflows" / workflow).read_text() + for job in ("Collect", "SecurityScan"): + with self.subTest(workflow=workflow, job=job): + self.assertRegex( + contents, + rf"(?m)^ {job}:\n runs-on: ubuntu-24\.04$", + ) + def test_missing_series_fails_without_publishing_outputs(self): self.manifest = [r for r in self.manifest if not r["version"].startswith("3.13.")] for workflow in WORKFLOWS: diff --git a/upload.sh b/upload.sh index f0aacf6..81db90f 100755 --- a/upload.sh +++ b/upload.sh @@ -5,7 +5,12 @@ set -euo pipefail : "${NEXUS_USER:?}" : "${NEXUS_PASSWORD:?}" -CA_BUNDLE="${CA_BUNDLE:-/etc/ssl/certs/ca-certificates.crt}" +if [[ ${CA_BUNDLE+x} ]]; then + echo "CA_BUNDLE is set: ${CA_BUNDLE}" +else + CA_BUNDLE="${CA_BUNDLE:-/etc/ssl/cert.pem}" + echo "Using CA_BUNDLE: ${CA_BUNDLE}" +fi command -v twine >/dev/null || { echo "ERROR: twine not found" @@ -18,16 +23,17 @@ REPORT="upload.log" : > "${REPORT}" [[ -r "${CA_BUNDLE}" ]] || { - echo "ERROR: CA bundle is not readable: ${CA_BUNDLE}" >&2 - exit 1 + echo "ERROR: CA bundle is not readable: ${CA_BUNDLE}" >&2 + exit 1 } -export TWINE_CA_BUNDLE="${CA_BUNDLE}" +export TWINE_CERT="${CA_BUNDLE}" +export REQUESTS_CA_BUNDLE="${CA_BUNDLE}" echo -en "\nFinding files to upload: " -find pypi \ - -type f \ - \( -name "*.whl" -o -name "*.tar.gz" -o -name "*.zip" \) \ - | sort > upload.lst +find pypi-* \ +-type f \ +\( -name "*.whl" -o -name "*.tar.gz" -o -name "*.zip" \) \ +| sort > upload.lst TOTAL=$(wc -l < upload.lst) echo "Done. Found ${TOTAL} files." @@ -35,11 +41,11 @@ echo -e "\nUploading files:" while IFS= read -r FILE do if twine upload \ - --repository-url "${NEXUS_URL}" \ - --username "${NEXUS_USER}" \ - --password "${NEXUS_PASSWORD}" \ - "${FILE}" \ - >> "${REPORT}" 2>&1 + --repository-url "${NEXUS_URL}" \ + --username "${NEXUS_USER}" \ + --password "${NEXUS_PASS}" \ + "${FILE}" \ + >> "${REPORT}" 2>&1 then echo "SUCCESS: ${FILE}" | tee -a "${REPORT}" else @@ -62,4 +68,4 @@ echo -e "\nUpload complete. See ${REPORT} for details." if [ "${FAILED}" -gt 0 ]; then exit 1 -fi \ No newline at end of file +fi