diff --git a/.github/actions/parse-version/script.php b/.github/actions/parse-version/script.php index e4875104..a4c0d5fd 100644 --- a/.github/actions/parse-version/script.php +++ b/.github/actions/parse-version/script.php @@ -105,6 +105,59 @@ function container_image_name($moodle_branch, $php) { return "ghcr.io/catalyst/$repo:latest"; } +/** + * Resolve a MOODLE_XXX_STABLE branch to a Moodle minor (10 digit) version. + * + * @param string $moodleBranch + * @param array $updates + * @return string + */ +function moodle_minor_version($moodleBranch, array $updates): string { + if (!preg_match('/^MOODLE_(\d+)_STABLE$/', $moodleBranch, $matches)) { + return ''; + } + + $series = $matches[1]; + $major = substr($series, 0, 1); + // Cast via int to strip leading zeros, then back to string for consistent branch key comparisons. + // This normalizes values used in the Moodle updates API map keys. + // Example: MOODLE_401_STABLE => 4.1, MOODLE_3011_STABLE => 3.11. + $minor = (string)((int)substr($series, 1)); + + $branch = $major . '.' . $minor; + foreach ($updates as $update) { + if (($update['branch'] ?? '') === $branch) { + return (string)($update['version'] ?? ''); + } + } + + return ''; +} + +/** + * Build a deterministic pgseed image reference for a matrix entry. + * + * @param array $entry + * @return string + */ +function pgseed_image_name(array $entry): string { + if (($entry['database'] ?? '') !== 'pgsql') { + return ''; + } + + $minor = $entry['moodle-minor'] ?? ''; + $pgsql = $entry['pgsql-ver'] ?? ''; + $php = $entry['php'] ?? ''; + $branch = $entry['moodle-branch'] ?? ''; + if ($minor === '' || $pgsql === '' || $php === '' || $branch === '') { + return ''; + } + + $branchSlug = str_replace('_', '-', strtolower($branch)); + $tag = "{$branchSlug}-php{$php}-pg{$pgsql}-m{$minor}"; + return "ghcr.io/catalyst/catalyst-moodle-workflows-pgseed:{$tag}"; +} + $preparedMatrix = array_filter($matrix['include'], function($entry) use($plugin, $updates, $matrix) { if (!isset($entry)) { @@ -205,9 +258,11 @@ function container_image_name($moodle_branch, $php) { // Add container image name and short branch name to each entry -$finalMatrix = array_map(function($entry) { +$finalMatrix = array_map(function($entry) use ($updates) { $entry['container'] = container_image_name($entry['moodle-branch'], $entry['php']); $entry['moodle-branch-short'] = preg_replace('/MOODLE_(.*)_STABLE/', '$1', $entry['moodle-branch']); + $entry['moodle-minor'] = moodle_minor_version($entry['moodle-branch'], $updates); + $entry['pgseed-image'] = pgseed_image_name($entry); return $entry; }, array_values($preparedMatrix)); diff --git a/.github/plugin/setup/action.yml b/.github/plugin/setup/action.yml index 7195b2b6..6772cc84 100644 --- a/.github/plugin/setup/action.yml +++ b/.github/plugin/setup/action.yml @@ -107,6 +107,36 @@ runs: ((test -f plugin/patch/${{ inputs.moodle_branch }}.diff && cd $GITHUB_WORKSPACE/moodletemp && git am --whitespace=nowarn < ../plugin/patch/${{ inputs.moodle_branch }}.diff) || echo No patch found;) shell: bash + - name: Apply MDL-88495 phpunit snapshot patch + if: ${{ always() }} + run: | + set -euo pipefail + cd "$GITHUB_WORKSPACE/moodletemp" + if grep -q -- "--upgrade" public/admin/tool/phpunit/cli/util.php; then + echo "MDL-88495 patch already present in this Moodle checkout." + exit 0 + fi + # Temporary upstream patch for MDL-88495 phpunit snapshot/restore/upgrade support. + patch_url="https://github.com/moodle/moodle/commit/e812521f260b571f05ced5efd3bb443a455bbf78.patch" + patch_file="/tmp/mdl88495-phpunitci.patch" + # SHA256 for commit patch e812521f260b571f05ced5efd3bb443a455bbf78. + expected_sha256="4aee5364e688fb22170bc902887c623439f00cf45baad54e8dedfd6e66594290" + curl -fsSL "$patch_url" -o "$patch_file" + actual_sha256="$(sha256sum "$patch_file" | awk '{print $1}')" + if [ "$actual_sha256" != "$expected_sha256" ]; then + echo "Patch checksum mismatch for MDL-88495 patch." + echo "Expected: $expected_sha256" + echo "Actual: $actual_sha256" + exit 1 + fi + if ! git am --3way --whitespace=nowarn /tmp/mdl88495-phpunitci.patch; then + git am --abort || true + echo "MDL-88495 patch did not apply cleanly on ${MOODLE_BRANCH}; continuing without patch (PHPUnit snapshot/restore features unavailable)." + fi + shell: bash + env: + MOODLE_BRANCH: ${{ inputs.moodle_branch }} + - name: Install Moodle and Plugin if: ${{ inputs.database != '' }} # Install moodle, but use our temporary directory to include any potential core patches that have been applied. @@ -123,4 +153,4 @@ runs: shell: bash env: DB: ${{ inputs.database }} - MOODLE_BRANCH: ${{ inputs.moodle_branch }} \ No newline at end of file + MOODLE_BRANCH: ${{ inputs.moodle_branch }} diff --git a/.github/workflows/build-pgseed.yml b/.github/workflows/build-pgseed.yml new file mode 100644 index 00000000..ddd3cad3 --- /dev/null +++ b/.github/workflows/build-pgseed.yml @@ -0,0 +1,147 @@ +name: Build pgseed images + +on: + workflow_dispatch: + schedule: + # Runs weekly (every Monday at 06:15 UTC) to refresh tags when Moodle minor versions roll forward. + - cron: '15 6 * * 1' + push: + branches: + - main + paths: + - '.github/actions/matrix/matrix_includes.yml' + - 'docker/pgseed/**' + - '.github/workflows/build-pgseed.yml' + +permissions: + contents: read + +jobs: + build-matrix: + runs-on: ubuntu-latest + outputs: + matrix: ${{ steps.set-matrix.outputs.matrix }} + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.x' + + - name: Install PyYAML + run: pip install pyyaml + + - name: Build pgseed matrix + id: set-matrix + run: | + import json + import os + import sys + import urllib.request + import yaml + + with open('.github/actions/matrix/matrix_includes.yml', encoding='utf-8') as f: + data = yaml.safe_load(f) + + MOODLE_UPDATES_QUERY_BRANCH = '3.8' + # branch=3.8 is accepted by this endpoint and still returns the full core updates list. + # version=0.0 requests the full available set rather than filtering to a real installed version. + updates_url = f'https://download.moodle.org/api/1.3/updates.php?format=json&version=0.0&branch={MOODLE_UPDATES_QUERY_BRANCH}' + try: + with urllib.request.urlopen(updates_url, timeout=30) as response: + updates_data = json.loads(response.read().decode('utf-8')) + except Exception as exc: + print(f'Failed to fetch Moodle updates from {updates_url}: {exc}', file=sys.stderr) + print('Please retry the workflow or run it manually once the API is reachable.', file=sys.stderr) + raise + + branch_to_minor = { + row['branch']: str(row['version']) + for row in updates_data.get('updates', {}).get('core', []) + if isinstance(row, dict) and row.get('branch') and row.get('version') + } + + def to_minor_branch(moodle_branch): + if not moodle_branch.startswith('MOODLE_') or not moodle_branch.endswith('_STABLE'): + return '' + digits = moodle_branch[len('MOODLE_'):-len('_STABLE')] + if not digits.isdigit() or len(digits) < 2: + return '' + major = digits[0] + minor = digits[1:].lstrip('0') or '0' + return f"{major}.{minor}" + + tuples = {} + for row in data.get('include', []): + if row.get('database') != 'pgsql': + continue + moodle_branch = row.get('moodle-branch', '') + minor_branch = to_minor_branch(moodle_branch) + moodle_minor = branch_to_minor.get(minor_branch, '') + if not moodle_minor: + # main does not have a stable 10-digit Moodle minor. + continue + + php = str(row.get('php', '')) + pgsql_ver = str(row.get('pgsql-ver', '')) + if not php or not pgsql_ver: + continue + + branch_slug = moodle_branch.replace('_', '-').lower() + image = 'ghcr.io/catalyst/catalyst-moodle-workflows-pgseed' + immutable_tag = f"{branch_slug}-php{php}-pg{pgsql_ver}-m{moodle_minor}" + mutable_tag = f"{branch_slug}-php{php}-pg{pgsql_ver}-latest" + + key = (moodle_branch, php, pgsql_ver, moodle_minor) + tuples[key] = { + 'moodle-branch': moodle_branch, + 'php': php, + 'pgsql-ver': pgsql_ver, + 'moodle-minor': moodle_minor, + 'image': image, + 'immutable-tag': immutable_tag, + 'mutable-tag': mutable_tag, + } + + matrix = sorted(tuples.values(), key=lambda x: (x['moodle-branch'], x['php'], x['pgsql-ver'])) + with open(os.environ['GITHUB_OUTPUT'], 'a', encoding='utf-8') as out: + out.write(f"matrix={json.dumps({'include': matrix})}\n") + shell: python + + build: + needs: build-matrix + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + strategy: + fail-fast: false + matrix: ${{ fromJson(needs.build-matrix.outputs.matrix) }} + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Log in to GHCR + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Build pgseed image + run: | + docker build \ + --build-arg MOODLE_BRANCH=${{ matrix['moodle-branch'] }} \ + --build-arg MOODLE_MINOR=${{ matrix['moodle-minor'] }} \ + --build-arg PHP_VERSION=${{ matrix.php }} \ + --build-arg POSTGRES_VERSION=${{ matrix['pgsql-ver'] }} \ + -t ${{ matrix.image }}:${{ matrix['immutable-tag'] }} \ + -t ${{ matrix.image }}:${{ matrix['mutable-tag'] }} \ + -f docker/pgseed/Dockerfile docker/pgseed + + - name: Push pgseed image tags + run: | + docker push ${{ matrix.image }}:${{ matrix['immutable-tag'] }} + docker push ${{ matrix.image }}:${{ matrix['mutable-tag'] }} diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a5a4afd5..1ec54294 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -446,7 +446,7 @@ jobs: runs-on: 'ubuntu-latest' services: postgres: - image: "postgres:${{ matrix.pgsql-ver }}" + image: "${{ matrix.database == 'pgsql' && matrix['pgseed-image'] != '' && matrix['pgseed-image'] || format('postgres:{0}', matrix['pgsql-ver']) }}" env: POSTGRES_USER: 'postgres' POSTGRES_HOST_AUTH_METHOD: 'trust' @@ -490,11 +490,29 @@ jobs: database: ${{ matrix.database }} - name: Run phpunit run: | + if [ "${MATRIX_DATABASE}" = "pgsql" ] && [ -n "${PGSEED_IMAGE}" ]; then + # Restore moodle sitedata from the pgseed postgres service container. + pgseed_container="$(docker ps --filter "name=postgres" --format '{{.ID}}' | head -n 1)" + if [ -n "$pgseed_container" ] && dataroot="$(php -r 'require "moodle/config.php"; if (empty($CFG->dataroot)) { exit(1); } echo $CFG->dataroot;' 2>/dev/null)"; then + if docker cp "$pgseed_container:/pgseed/moodledata.tar.gz" /tmp/moodledata.tar.gz 2>/dev/null; then + mkdir -p "$dataroot" + tar -xzf /tmp/moodledata.tar.gz -C "$dataroot" --strip-components=1 + fi + fi + fi + if php moodle/public/admin/tool/phpunit/cli/util.php --help | grep -q -- '--upgrade'; then + php moodle/public/admin/tool/phpunit/cli/util.php --upgrade + else + echo "PHPUnit util upgrade mode not available on this Moodle branch; continuing with standard PHPUnit flow." + fi moodle-plugin-ci phpunit cd moodle vendor/bin/phpunit --fail-on-risky --disallow-test-output --filter tool_dataprivacy_metadata_registry_testcase vendor/bin/phpunit --fail-on-risky --disallow-test-output --testsuite core_privacy_testsuite --filter provider_testcase shell: bash + env: + MATRIX_DATABASE: ${{ matrix.database }} + PGSEED_IMAGE: ${{ matrix['pgseed-image'] }} behat: name: ${{ matrix.moodle-branch-short }} - behat - php${{ matrix.php }} - ${{ matrix.database }} needs: prepare_matrix @@ -510,7 +528,7 @@ jobs: runs-on: 'ubuntu-latest' services: postgres: - image: "postgres:${{ matrix.pgsql-ver }}" + image: "${{ matrix.database == 'pgsql' && matrix['pgseed-image'] != '' && matrix['pgseed-image'] || format('postgres:{0}', matrix['pgsql-ver']) }}" env: POSTGRES_USER: 'postgres' POSTGRES_HOST_AUTH_METHOD: 'trust' diff --git a/README.md b/README.md index a2743676..6b5bf0c7 100644 --- a/README.md +++ b/README.md @@ -142,6 +142,17 @@ When you call the reusable ci, it will: To view or modify the full matrix, please see it here: [.github/actions/matrix/matrix_includes.yml](.github/actions/matrix/matrix_includes.yml) +### Seeded PostgreSQL images + +PostgreSQL-backed CI jobs can use pre-built pgseed images published to GHCR: + +`ghcr.io/catalyst/catalyst-moodle-workflows-pgseed:-php-pg-m` + +- `` is the Moodle 10-digit minor version from the Moodle updates API (same source used by matrix parsing). +- Images are built by `.github/workflows/build-pgseed.yml` from `docker/pgseed/`. +- Rebuild these images by running the `Build pgseed images` workflow manually, or by changing `docker/pgseed/**`, `.github/actions/matrix/matrix_includes.yml`, or `.github/workflows/build-pgseed.yml`. +- CI applies the temporary MDL-88495 patch to the Moodle checkout (until merged upstream), runs `php public/admin/tool/phpunit/cli/util.php --upgrade` before phpunit, and restores sitedata from the pgseed postgres service container archive at `/pgseed/moodledata.tar.gz` (docker-level snapshot, no `util.php --restore` usage). + ## How does this automate releases? Whenever a change is made to version.php, the workflow is triggered on a release branch (e.g. __main__ / __MOODLE_XX_STABLE__), and tests pass, will it attempt to run the plugin/release action `.github/plugin/release/action.yml`. Doing so will automatically publish a release to the Moodle plugin directory at https://moodle.org/plugins. diff --git a/docker/pgseed/Dockerfile b/docker/pgseed/Dockerfile new file mode 100644 index 00000000..020682ad --- /dev/null +++ b/docker/pgseed/Dockerfile @@ -0,0 +1,26 @@ +ARG POSTGRES_VERSION=15 +FROM postgres:${POSTGRES_VERSION} + +ARG MOODLE_BRANCH +ARG MOODLE_MINOR +ARG PHP_VERSION +ARG POSTGRES_VERSION + +ENV MOODLE_BRANCH=${MOODLE_BRANCH} +ENV MOODLE_MINOR=${MOODLE_MINOR} +ENV PHP_VERSION=${PHP_VERSION} +ENV POSTGRES_VERSION=${POSTGRES_VERSION} + +LABEL org.opencontainers.image.source="https://github.com/catalyst/catalyst-moodle-workflows" +LABEL org.opencontainers.image.description="Seeded PostgreSQL image for catalyst-moodle-workflows" +LABEL org.catalyst.moodle.branch="${MOODLE_BRANCH}" +LABEL org.catalyst.moodle.minor="${MOODLE_MINOR}" +LABEL org.catalyst.php.version="${PHP_VERSION}" +LABEL org.catalyst.pgsql.version="${POSTGRES_VERSION}" + +COPY seed.sh /docker-entrypoint-initdb.d/20-seed.sh +COPY moodledata/ /opt/pgseed/moodledata/ +RUN chmod +x /docker-entrypoint-initdb.d/20-seed.sh +RUN mkdir -p /pgseed \ + && tar -C /opt/pgseed -czf /pgseed/moodledata.tar.gz moodledata \ + && rm -rf /opt/pgseed diff --git a/docker/pgseed/moodledata/.gitkeep b/docker/pgseed/moodledata/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/docker/pgseed/seed.sh b/docker/pgseed/seed.sh new file mode 100644 index 00000000..4534ae7c --- /dev/null +++ b/docker/pgseed/seed.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash +set -euo pipefail + +MOODLE_BRANCH="${MOODLE_BRANCH:-unknown}" +MOODLE_MINOR="${MOODLE_MINOR:-unknown}" +PHP_VERSION="${PHP_VERSION:-unknown}" +POSTGRES_VERSION="${POSTGRES_VERSION:-unknown}" + +psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" <<-SQL + CREATE DATABASE moodle; +SQL + +psql \ + -v ON_ERROR_STOP=1 \ + -v moodle_branch="$MOODLE_BRANCH" \ + -v moodle_minor="$MOODLE_MINOR" \ + -v php_version="$PHP_VERSION" \ + -v pgsql_version="$POSTGRES_VERSION" \ + --username "$POSTGRES_USER" \ + --dbname moodle <<-'SQL' + CREATE SCHEMA IF NOT EXISTS ci_seed; + CREATE TABLE IF NOT EXISTS ci_seed.metadata ( + id BIGSERIAL PRIMARY KEY, + moodle_branch TEXT NOT NULL, + moodle_minor TEXT NOT NULL, + php_version TEXT NOT NULL, + pgsql_version TEXT NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW() + ); + INSERT INTO ci_seed.metadata (moodle_branch, moodle_minor, php_version, pgsql_version) + VALUES (:'moodle_branch', :'moodle_minor', :'php_version', :'pgsql_version'); +SQL