diff --git a/.github/actions/matrix/matrix_includes.yml b/.github/actions/matrix/matrix_includes.yml index 2639fe8f..a4fba8bc 100644 --- a/.github/actions/matrix/matrix_includes.yml +++ b/.github/actions/matrix/matrix_includes.yml @@ -1,26 +1,26 @@ include: # Always include main (issue #18) - disable-able by config by setting 'disable_master' to true - - {moodle-branch: 'main', php: '8.4', node: '22', mariadb-ver: '10.11', pgsql-ver: '16', database: 'pgsql'} + - {moodle-branch: 'main', php: '8.4', node: '22', pgsql-ver: '16', database: 'pgsql'} # Test all variants of 5.2 - - { moodle-branch: 'MOODLE_502_STABLE', php: '8.4', node: '22', mariadb-ver: '10.11', pgsql-ver: '16', database: 'mariadb' } - - { moodle-branch: 'MOODLE_502_STABLE', php: '8.4', node: '22', mariadb-ver: '10.11', pgsql-ver: '16', database: 'pgsql' } - - { moodle-branch: 'MOODLE_502_STABLE', php: '8.3', node: '22', mariadb-ver: '10.11', pgsql-ver: '16', database: 'mariadb' } - - { moodle-branch: 'MOODLE_502_STABLE', php: '8.3', node: '22', mariadb-ver: '10.11', pgsql-ver: '16', database: 'pgsql' } + - { moodle-branch: 'MOODLE_502_STABLE', php: '8.4', node: '22', mariadb-ver: '10.11', database: 'mariadb' } + - { moodle-branch: 'MOODLE_502_STABLE', php: '8.4', node: '22', pgsql-ver: '16', database: 'pgsql' } + - { moodle-branch: 'MOODLE_502_STABLE', php: '8.3', node: '22', mariadb-ver: '10.11', database: 'mariadb' } + - { moodle-branch: 'MOODLE_502_STABLE', php: '8.3', node: '22', pgsql-ver: '16', database: 'pgsql' } # Test all variants of 5.1 - - { moodle-branch: 'MOODLE_501_STABLE', php: '8.4', node: '22', mariadb-ver: '10.11', pgsql-ver: '15', database: 'mariadb' } - - { moodle-branch: 'MOODLE_501_STABLE', php: '8.4', node: '22', mariadb-ver: '10.11', pgsql-ver: '15', database: 'pgsql' } - - { moodle-branch: 'MOODLE_501_STABLE', php: '8.2', node: '22', mariadb-ver: '10.11', pgsql-ver: '15', database: 'mariadb' } - - { moodle-branch: 'MOODLE_501_STABLE', php: '8.2', node: '22', mariadb-ver: '10.11', pgsql-ver: '15', database: 'pgsql' } + - { moodle-branch: 'MOODLE_501_STABLE', php: '8.4', node: '22', mariadb-ver: '10.11', database: 'mariadb' } + - { moodle-branch: 'MOODLE_501_STABLE', php: '8.4', node: '22', pgsql-ver: '15', database: 'pgsql' } + - { moodle-branch: 'MOODLE_501_STABLE', php: '8.2', node: '22', mariadb-ver: '10.11', database: 'mariadb' } + - { moodle-branch: 'MOODLE_501_STABLE', php: '8.2', node: '22', pgsql-ver: '15', database: 'pgsql' } # Test limited variants of 5.0 - - { moodle-branch: 'MOODLE_500_STABLE', php: '8.4', node: '22', mariadb-ver: '10.11', pgsql-ver: '14', database: 'mariadb' } - - { moodle-branch: 'MOODLE_500_STABLE', php: '8.4', node: '22', mariadb-ver: '10.11', pgsql-ver: '14', database: 'pgsql' } + - { moodle-branch: 'MOODLE_500_STABLE', php: '8.4', node: '22', mariadb-ver: '10.11', database: 'mariadb' } + - { moodle-branch: 'MOODLE_500_STABLE', php: '8.4', node: '22', pgsql-ver: '14', database: 'pgsql' } # Test all variants of 4.5. (LTS). - - { moodle-branch: 'MOODLE_405_STABLE', php: '8.3', node: '22', mariadb-ver: '10.6', pgsql-ver: '13', database: 'mariadb' } - - { moodle-branch: 'MOODLE_405_STABLE', php: '8.3', node: '22', mariadb-ver: '10.6', pgsql-ver: '13', database: 'pgsql' } - - { moodle-branch: 'MOODLE_405_STABLE', php: '8.1', node: '22', mariadb-ver: '10.6', pgsql-ver: '13', database: 'mariadb' } - - { moodle-branch: 'MOODLE_405_STABLE', php: '8.1', node: '22', mariadb-ver: '10.6', pgsql-ver: '13', database: 'pgsql' } + - { moodle-branch: 'MOODLE_405_STABLE', php: '8.3', node: '22', mariadb-ver: '10.6', database: 'mariadb' } + - { moodle-branch: 'MOODLE_405_STABLE', php: '8.3', node: '22', pgsql-ver: '13', database: 'pgsql' } + - { moodle-branch: 'MOODLE_405_STABLE', php: '8.1', node: '22', mariadb-ver: '10.6', database: 'mariadb' } + - { moodle-branch: 'MOODLE_405_STABLE', php: '8.1', node: '22', pgsql-ver: '13', database: 'pgsql' } # Test all variants of 4.1 (LTS). - - { moodle-branch: 'MOODLE_401_STABLE', php: '7.4', node: '22', mariadb-ver: '10.5', pgsql-ver: '12', database: 'mariadb' } - - { moodle-branch: 'MOODLE_401_STABLE', php: '7.4', node: '22', mariadb-ver: '10.5', pgsql-ver: '12', database: 'pgsql' } - - { moodle-branch: 'MOODLE_401_STABLE', php: '8.0', node: '22', mariadb-ver: '10.5', pgsql-ver: '12', database: 'mariadb' } - - { moodle-branch: 'MOODLE_401_STABLE', php: '8.0', node: '22', mariadb-ver: '10.5', pgsql-ver: '12', database: 'pgsql' } + - { moodle-branch: 'MOODLE_401_STABLE', php: '7.4', node: '22', mariadb-ver: '10.5', database: 'mariadb' } + - { moodle-branch: 'MOODLE_401_STABLE', php: '7.4', node: '22', pgsql-ver: '12', database: 'pgsql' } + - { moodle-branch: 'MOODLE_401_STABLE', php: '8.0', node: '22', mariadb-ver: '10.5', database: 'mariadb' } + - { moodle-branch: 'MOODLE_401_STABLE', php: '8.0', node: '22', pgsql-ver: '12', database: 'pgsql' } diff --git a/.github/actions/parse-version/script.php b/.github/actions/parse-version/script.php index 75f86427..84595efb 100644 --- a/.github/actions/parse-version/script.php +++ b/.github/actions/parse-version/script.php @@ -248,22 +248,35 @@ function container_image_name($moodle_branch, $php) { usort($phpVersions, 'version_compare'); $highestPhp = end($phpVersions); output('highest_php', $highestPhp); - // Find the container and node for this combo + // Find the container and node for this combo. + // Prefer the pgsql entry when present so latest_pgsql_ver is reliably populated. $foundNode = ''; + $foundPgsqlVer = ''; foreach ($finalMatrix as $entry) { if ($entry['moodle-branch'] === $highestMoodleBranch && $entry['php'] === $highestPhp) { output('latest_container', $entry['container']); - if (isset($entry['pgsql-ver'])) { - output('latest_pgsql_ver', $entry['pgsql-ver']); - } else { - output('latest_pgsql_ver', ''); - } if (isset($entry['node'])) { $foundNode = $entry['node']; } - break; + if (($entry['database'] ?? '') === 'pgsql' && isset($entry['pgsql-ver'])) { + $foundPgsqlVer = $entry['pgsql-ver']; + break; + } + } + } + + // If no exact highest-php pgsql entry was found, fallback to any pgsql for the highest branch. + if ($foundPgsqlVer === '') { + foreach ($finalMatrix as $entry) { + if ($entry['moodle-branch'] === $highestMoodleBranch && ($entry['database'] ?? '') === 'pgsql' && isset($entry['pgsql-ver'])) { + $foundPgsqlVer = $entry['pgsql-ver']; + break; + } } } + + output('latest_pgsql_ver', $foundPgsqlVer); + // If node version not found above, fallback to highest node version if (!$foundNode && !empty($nodeVersions)) { usort($nodeVersions, 'version_compare'); diff --git a/.github/patches/401-403-phpunit-restore.patch b/.github/patches/401-403-phpunit-restore.patch new file mode 100644 index 00000000..6d5b14b7 --- /dev/null +++ b/.github/patches/401-403-phpunit-restore.patch @@ -0,0 +1,498 @@ +From 922f91b1cfd353c00f5dcd41d7c52bced26917b4 Mon Sep 17 00:00:00 2001 +From: Abhinav Gandham +Date: Fri, 22 May 2026 15:05:47 +1000 +Subject: [PATCH] MDL-88495 Unit tests: Snapshot and restore for test site. + +--- + admin/tool/phpunit/cli/util.php | 28 ++- + lib/phpunit/bootstrap.php | 2 +- + lib/phpunit/classes/util.php | 376 +++++++++++++++++++++++++++++++- + lib/testing/classes/util.php | 7 + + 4 files changed, 408 insertions(+), 5 deletions(-) + +diff --git a/admin/tool/phpunit/cli/util.php b/admin/tool/phpunit/cli/util.php +index 1d176900fdf..39cd67fb20f 100644 +--- a/admin/tool/phpunit/cli/util.php ++++ b/admin/tool/phpunit/cli/util.php +@@ -47,6 +47,9 @@ list($options, $unrecognized) = cli_get_params( + 'buildcomponentconfigs' => false, + 'diag' => false, + 'run' => false, ++ 'upgrade' => false, ++ 'snapshot' => '', ++ 'restore' => '', + 'help' => false, + ), + array( +@@ -102,8 +105,16 @@ $drop = $options['drop']; + $install = $options['install']; + $buildconfig = $options['buildconfig']; + $buildcomponentconfigs = $options['buildcomponentconfigs']; +- +-if ($options['help'] or (!$drop and !$install and !$buildconfig and !$buildcomponentconfigs and !$diag)) { ++$snapshot = $options['snapshot']; ++$restore = $options['restore']; ++$upgrade = $options['upgrade']; ++ ++if ( ++ $options['help'] || ( ++ !$drop && !$install && !$buildconfig && !$buildcomponentconfigs ++ && !$diag && !$upgrade && !$snapshot && !$restore ++ ) ++) { + $help = "Various PHPUnit utility functions + + Options: +@@ -114,6 +125,10 @@ Options: + --buildconfig Build /phpunit.xml from /phpunit.xml.dist that runs all tests + --buildcomponentconfigs + Build distributed phpunit.xml files for each component ++--upgrade Upgrade test site to latest version ++--snapshot Create snapshot of the current test site. Optionally specify name of the snapshot with ++ --snapshot=NAME. ++--restore=NAME Restore snapshot of test site with the specified name. + + -h, --help Print out this help + +@@ -152,4 +167,13 @@ if ($diag) { + } else if ($install) { + phpunit_util::install_site(); + exit(0); ++} else if ($upgrade) { ++ phpunit_util::upgrade_site(); ++ exit(0); ++} else if ($snapshot) { ++ phpunit_util::snapshot_site(is_string($snapshot) ? $snapshot : ''); ++ exit(0); ++} else if ($restore) { ++ phpunit_util::restore_site(is_string($restore) ? $restore : ''); ++ exit(0); + } +diff --git a/lib/phpunit/bootstrap.php b/lib/phpunit/bootstrap.php +index e8be0752c2d..d1306ff2c54 100644 +--- a/lib/phpunit/bootstrap.php ++++ b/lib/phpunit/bootstrap.php +@@ -135,7 +135,7 @@ if (!is_writable($CFG->phpunit_dataroot)) { + if (!file_exists("$CFG->phpunit_dataroot/phpunittestdir.txt")) { + if ($dh = opendir($CFG->phpunit_dataroot)) { + while (($file = readdir($dh)) !== false) { +- if ($file === 'phpunit' or $file === '.' or $file === '..' or $file === '.DS_Store') { ++ if ($file === 'phpunit' || $file === '.' || $file === '..' || $file === '.DS_Store' || str_ends_with($file, '.zip')) { + continue; + } + phpunit_bootstrap_error(PHPUNIT_EXITCODE_CONFIGERROR, '$CFG->phpunit_dataroot directory is not empty, can not run tests! Is it used for anything else?'); +diff --git a/lib/phpunit/classes/util.php b/lib/phpunit/classes/util.php +index 9d5acc4bae0..44a3215e1f5 100644 +--- a/lib/phpunit/classes/util.php ++++ b/lib/phpunit/classes/util.php +@@ -427,6 +427,13 @@ class phpunit_util extends testing_util { + echo "Purging dataroot:\n"; + } + ++ // Preserve any snapshot zips so they survive the drop. ++ foreach (scandir(self::get_dataroot()) as $item) { ++ if (str_ends_with($item, '.zip')) { ++ static::$datarootskiponreset[] = $item; ++ } ++ } ++ + self::reset_dataroot(); + testing_initdataroot($CFG->dataroot, 'phpunit'); + +@@ -496,8 +503,373 @@ class phpunit_util extends testing_util { + } + + /** +- * Builds dirroot/phpunit.xml file using defaults from /phpunit.xml.dist +- * @static ++ * Create a snapshot of the current test site database and site data. ++ * ++ * @param string $snapshotname The name of the snapshot. ++ * @return void may terminate execution with exit code ++ */ ++ public static function snapshot_site(string $snapshotname) { ++ global $DB, $CFG; ++ ++ require($CFG->dirroot . '/version.php'); ++ ++ if (!self::is_test_site()) { ++ phpunit_bootstrap_error(PHPUNIT_EXITCODE_CONFIGERROR, 'Can not snapshot non-test site!!'); ++ } ++ ++ if (empty($DB->get_tables())) { ++ phpunit_bootstrap_error(PHPUNIT_EXITCODE_INSTALL, 'No database tables present, can not snapshot site!!'); ++ } ++ ++ $dataroot = self::get_dataroot(); ++ $dbtype = $CFG->dbtype; ++ $name = $snapshotname ? "$dbtype-$snapshotname-$version" : "$dbtype-snapshot-$version"; ++ $snapshotdir = $dataroot . "/$name"; ++ ++ // Clear old in-memory caches of snapshot data. ++ self::$tabledata = null; ++ self::$tablestructure = null; ++ self::$sequencenames = null; ++ ++ $sitedatapath = $dataroot . DIRECTORY_SEPARATOR . self::get_originaldatafilesjson(); ++ ++ if (file_exists($sitedatapath)) { ++ unlink($sitedatapath); ++ } ++ ++ self::reset_original_data(); ++ self::save_original_data_files(); ++ self::store_database_state(); ++ self::store_versions_hash(); ++ self::$lastdbwrites = $DB->perf_get_writes(); ++ ++ if (is_dir($snapshotdir)) { ++ remove_dir($snapshotdir); ++ } ++ ++ mkdir($snapshotdir); ++ ++ $filedir = $dataroot . DIRECTORY_SEPARATOR . 'filedir'; ++ ++ self::copy_filedir($filedir, $snapshotdir . DIRECTORY_SEPARATOR . 'filedir' . DIRECTORY_SEPARATOR); ++ ++ $datafiles = [ ++ "$dataroot" . DIRECTORY_SEPARATOR . "phpunit" . DIRECTORY_SEPARATOR . "tabledata.ser", ++ "$dataroot" . DIRECTORY_SEPARATOR . "phpunit" . DIRECTORY_SEPARATOR . "tablestructure.ser", ++ "$dataroot" . DIRECTORY_SEPARATOR . "phpunit" . DIRECTORY_SEPARATOR . "versionshash.txt", ++ "$dataroot" . DIRECTORY_SEPARATOR . "originaldatafiles.json", ++ ]; ++ ++ foreach ($datafiles as $file) { ++ if (!file_exists($file)) { ++ continue; ++ } ++ copy($file, $snapshotdir . DIRECTORY_SEPARATOR . basename($file)); ++ } ++ ++ $dirtozip = $dataroot . DIRECTORY_SEPARATOR . "$name"; ++ ++ $zip = new \ZipArchive(); ++ $zip->open("$dirtozip.zip", \ZipArchive::CREATE | \ZipArchive::OVERWRITE); ++ ++ $files = new \RecursiveIteratorIterator( ++ new \RecursiveDirectoryIterator($dirtozip, \RecursiveDirectoryIterator::SKIP_DOTS), ++ \RecursiveIteratorIterator::LEAVES_ONLY ++ ); ++ ++ foreach ($files as $file) { ++ if (!$file->isDir()) { ++ $filepath = $file->getRealPath(); ++ $filepathrelative = substr($filepath, strlen($dirtozip) + 1); ++ $zip->addFile($filepath, $filepathrelative); ++ } ++ } ++ ++ $zip->close(); ++ remove_dir($snapshotdir); ++ ++ echo "Snapshot successfully created in $dirtozip.zip\n"; ++ } ++ ++ /** ++ * Restore the database and dataroot to the state they were in when snapshot_site() was called. ++ * ++ * @param string $snapshot The snapshot to restore. ++ * @return void may terminate execution with exit code ++ */ ++ public static function restore_site(string $snapshot) { ++ global $DB, $CFG; ++ ++ require($CFG->dirroot . '/version.php'); ++ ++ if (!self::is_test_site()) { ++ phpunit_bootstrap_error(PHPUNIT_EXITCODE_CONFIGERROR, 'Can not restore non-test site!!'); ++ } ++ ++ if (!empty($DB->get_tables())) { ++ echo "Database tables are still present. Run php public/admin/tool/phpunit/cli/util.php --drop before restoring.\n"; ++ return; ++ } ++ ++ if (!$snapshot) { ++ echo "No snapshot name provided. Please provide the snapshot name as an argument.\n"; ++ return; ++ } ++ ++ $dataroot = self::get_dataroot(); ++ $zipfile = $dataroot . DIRECTORY_SEPARATOR . "$snapshot.zip"; ++ $destpath = $dataroot . DIRECTORY_SEPARATOR . "$snapshot"; ++ ++ $zip = new \ZipArchive(); ++ if ($zip->open($zipfile) === true) { ++ $zip->extractTo($destpath); ++ $zip->close(); ++ } else { ++ phpunit_bootstrap_error(PHPUNIT_EXITCODE_CONFIGERROR, "Can not open snapshot zip file $zipfile!!"); ++ } ++ ++ $snapshotdir = $dataroot . "/$snapshot"; ++ ++ // Map snapshot files to their active locations. ++ $datafiles = [ ++ "$snapshotdir" . DIRECTORY_SEPARATOR . "tabledata.ser" ++ => $dataroot . DIRECTORY_SEPARATOR . 'phpunit' . DIRECTORY_SEPARATOR . "tabledata.ser", ++ "$snapshotdir" . DIRECTORY_SEPARATOR . "tablestructure.ser" ++ => $dataroot . DIRECTORY_SEPARATOR . 'phpunit' . DIRECTORY_SEPARATOR . "tablestructure.ser", ++ "$snapshotdir" . DIRECTORY_SEPARATOR . "versionshash.txt" ++ => $dataroot . DIRECTORY_SEPARATOR . 'phpunit' . DIRECTORY_SEPARATOR . "versionshash.txt", ++ "$snapshotdir" . DIRECTORY_SEPARATOR . "originaldatafiles.json" ++ => $dataroot . DIRECTORY_SEPARATOR . self::get_originaldatafilesjson(), ++ ]; ++ ++ foreach ($datafiles as $src => $dest) { ++ if (!file_exists($src)) { ++ phpunit_bootstrap_error(PHPUNIT_EXITCODE_CONFIGERROR, "Required snapshot file $src does not exist!!"); ++ } ++ } ++ ++ self::$lastdbwrites = null; ++ self::$tabledata = null; ++ self::$tablestructure = null; ++ self::$sequencenames = null; ++ ++ echo "Restoring from $snapshot\n"; ++ ++ $phpunitdir = $dataroot . DIRECTORY_SEPARATOR . 'phpunit'; ++ if (!is_dir($phpunitdir)) { ++ mkdir($phpunitdir, 0777, true); ++ } ++ ++ // Copy snapshot DB control files to their active locations. ++ foreach ($datafiles as $src => $dest) { ++ copy($src, $dest); ++ } ++ ++ if (empty($DB->get_tables(false))) { ++ self::create_tables_from_snapshot($destpath); ++ } ++ ++ echo "Importing snapshot data...\n"; ++ // Restore the database to snapshot state. ++ self::reset_database(); ++ ++ echo "Data imported. Resetting datarooot...\n"; ++ ++ // Clear current dataroot before restoring filedir from snapshot. ++ static::$datarootskiponreset = ['.', '..', 'phpunittestdir.txt', 'phpunit', '.htaccess', $snapshot, "$snapshot.zip"]; ++ self::reset_dataroot(); ++ self::reset_original_data(); ++ ++ $snapshotfiledir = $snapshotdir . DIRECTORY_SEPARATOR . 'filedir'; ++ $filedir = $dataroot . DIRECTORY_SEPARATOR . 'filedir'; ++ ++ // Copy snapshot filedir back to dataroot. ++ self::copy_filedir($snapshotfiledir, $filedir); ++ ++ initialise_cfg(); ++ ++ // Execute all adhoc tasks. ++ while ($task = \core\task\manager::get_next_adhoc_task(time())) { ++ $task->execute(); ++ \core\task\manager::adhoc_task_complete($task); ++ } ++ ++ set_config('upgraderunning', 0); ++ ++ self::store_database_state(); ++ self::$lastdbwrites = $DB->perf_get_writes(); ++ ++ remove_dir($snapshotdir); ++ unlink($zipfile); ++ ++ echo "Snapshot successfully restored.\n"; ++ } ++ ++ /** ++ * Upgrade the test site and upgrade/install any new plugins that were added. ++ * ++ * @return void may terminate execution with exit code ++ */ ++ public static function upgrade_site() { ++ ++ if (!self::is_test_site()) { ++ phpunit_bootstrap_error(PHPUNIT_EXITCODE_CONFIGERROR, 'Can not upgrade non-test site!!'); ++ } ++ ++ if (self::is_test_data_updated()) { ++ echo "No database changes detected, skipping upgrade.\n"; ++ return; ++ } ++ ++ initialise_cfg(); ++ upgrade_noncore(true); ++ set_config('upgraderunning', 0); ++ self::store_versions_hash(); ++ self::store_database_state(); ++ } ++ ++ /** ++ * Helper method that copies all contents from one directory to another, creating directories as needed. ++ * ++ * @param string $sourcedir the source directory to copy ++ * @param string $dirtocreate the destination directory to create and copy files to ++ * @return void ++ */ ++ private static function copy_filedir($sourcedir, $dirtocreate) { ++ ++ if (!file_exists($sourcedir)) { ++ echo "Source directory $sourcedir does not exist!\n"; ++ return; ++ } ++ ++ $iterator = new \RecursiveIteratorIterator( ++ new \RecursiveDirectoryIterator($sourcedir, \RecursiveDirectoryIterator::SKIP_DOTS), ++ \RecursiveIteratorIterator::SELF_FIRST ++ ); ++ ++ if (!is_dir($dirtocreate)) { ++ mkdir($dirtocreate, 0777, true); ++ } ++ ++ foreach ($iterator as $item) { ++ $destpath = $dirtocreate . DIRECTORY_SEPARATOR . $iterator->getSubPathName(); ++ if ($item->isDir()) { ++ if (!is_dir($destpath)) { ++ mkdir($destpath, 0777, true); ++ } ++ } else { ++ copy($item->getPathname(), $destpath); ++ } ++ } ++ } ++ ++ /** ++ * Create the database tables based on the stored snapshot structure. ++ * ++ * @param string $snapshotdir the target snapshot directory to create the tables from. ++ * @param bool $showprogress whether to show progress during table creation. ++ * @return void may terminate execution with exit code ++ */ ++ private static function create_tables_from_snapshot(string $snapshotdir, bool $showprogress = true) { ++ global $DB; ++ ++ $dbman = $DB->get_manager(); ++ ++ $structurefile = $snapshotdir . DIRECTORY_SEPARATOR . 'tablestructure.ser'; ++ if (!file_exists($structurefile)) { ++ phpunit_bootstrap_error(PHPUNIT_EXITCODE_CONFIGERROR, "Snapshot file $structurefile does not exist!!"); ++ } ++ $tablestructure = unserialize(file_get_contents($structurefile)); ++ if (!is_array($tablestructure)) { ++ phpunit_bootstrap_error(PHPUNIT_EXITCODE_CONFIGERROR, "Invalid tablestructure.ser in snapshot $snapshotdir"); ++ } ++ ++ $dotsonline = 0; ++ foreach ($tablestructure as $tablename => $columns) { ++ // If the table already exists, skip it. ++ if ($dbman->table_exists($tablename)) { ++ continue; ++ } ++ ++ $table = new \xmldb_table($tablename); ++ ++ // Build an xmldb_field for every column. ++ foreach ($columns as $colname => $col) { ++ $field = new \xmldb_field($colname); ++ $sequence = false; ++ // Map each column meta type to the appropriate xmldb type. ++ switch ($col->meta_type) { ++ case 'R': ++ $field->setType(XMLDB_TYPE_INTEGER); ++ $field->setLength(10); ++ $sequence = true; ++ break; ++ case 'I': ++ $field->setType(XMLDB_TYPE_INTEGER); ++ $field->setLength($col->max_length ?: 10); ++ break; ++ case 'N': ++ $floattypes = ['float', 'double', 'float4', 'float8', 'double precision']; ++ if (in_array(strtolower($col->type ?? ''), $floattypes)) { ++ $field->setType(XMLDB_TYPE_FLOAT); ++ } else { ++ $field->setType(XMLDB_TYPE_NUMBER); ++ $field->setLength($col->max_length ?: 10); ++ $field->setDecimals($col->scale ?: 0); ++ } ++ break; ++ case 'C': ++ $field->setType(XMLDB_TYPE_CHAR); ++ $field->setLength($col->max_length ?: 255); ++ break; ++ case 'X': ++ $field->setType(XMLDB_TYPE_TEXT); ++ break; ++ case 'B': ++ $field->setType(XMLDB_TYPE_BINARY); ++ break; ++ case 'L': ++ $field->setType(XMLDB_TYPE_INTEGER); ++ $field->setLength(1); ++ break; ++ default: ++ $field->setType(XMLDB_TYPE_INTEGER); ++ $field->setLength($col->max_length ?: 10); ++ } ++ $field->setNotNull($col->not_null); ++ if ($col->has_default && $col->default_value !== '') { ++ $field->setDefault($col->default_value); ++ } ++ if ($sequence) { ++ $field->setSequence(true); ++ } ++ $table->addField($field); ++ } ++ if (isset($columns['id'])) { ++ // Add a primary key for the 'id' column. ++ $key = new \xmldb_key('primary', XMLDB_KEY_PRIMARY, ['id']); ++ $table->addKey($key); ++ } ++ $dbman->create_table($table); ++ ++ if ($dotsonline == 60) { ++ if ($showprogress) { ++ echo "\n"; ++ } ++ $dotsonline = 0; ++ } ++ if ($showprogress) { ++ echo '.'; ++ } ++ $dotsonline += 1; ++ } ++ ++ echo "\nTables successfully created from snapshot.\n"; ++ } ++ ++ /** ++ * Builds root/phpunit.xml file using defaults from /phpunit.xml.dist ++ * + * @return bool true means main config file created, false means only dataroot file created + */ + public static function build_config_file() { +diff --git a/lib/testing/classes/util.php b/lib/testing/classes/util.php +index 0c241e6d2b9..fec29319bb1 100644 +--- a/lib/testing/classes/util.php ++++ b/lib/testing/classes/util.php +@@ -136,6 +136,13 @@ abstract class testing_util { + return substr($classname, 0, strpos($classname, '_')); + } + ++ /** ++ * Reset the original data. ++ */ ++ protected static function reset_original_data() { ++ self::$originaldatafilesjsonadded = false; ++ } ++ + /** + * Get data generator + * @static +-- +2.43.0 + diff --git a/.github/patches/404-500-phpunit-restore.patch b/.github/patches/404-500-phpunit-restore.patch new file mode 100644 index 00000000..a490ca33 --- /dev/null +++ b/.github/patches/404-500-phpunit-restore.patch @@ -0,0 +1,498 @@ +From 192dc1a2146891462759923aaf6e6f934999c99a Mon Sep 17 00:00:00 2001 +From: Abhinav Gandham +Date: Fri, 22 May 2026 15:05:47 +1000 +Subject: [PATCH] MDL-88495 Unit tests: Snapshot and restore for test site. + +--- + admin/tool/phpunit/cli/util.php | 28 ++- + lib/phpunit/bootstrap.php | 2 +- + lib/phpunit/classes/util.php | 376 +++++++++++++++++++++++++++++++- + lib/testing/classes/util.php | 7 + + 4 files changed, 408 insertions(+), 5 deletions(-) + +diff --git a/admin/tool/phpunit/cli/util.php b/admin/tool/phpunit/cli/util.php +index 3cc928a9e5a..988e45f150d 100644 +--- a/admin/tool/phpunit/cli/util.php ++++ b/admin/tool/phpunit/cli/util.php +@@ -47,6 +47,9 @@ list($options, $unrecognized) = cli_get_params( + 'buildcomponentconfigs' => false, + 'diag' => false, + 'run' => false, ++ 'upgrade' => false, ++ 'snapshot' => '', ++ 'restore' => '', + 'help' => false, + ], + [ +@@ -102,8 +105,16 @@ $drop = $options['drop']; + $install = $options['install']; + $buildconfig = $options['buildconfig']; + $buildcomponentconfigs = $options['buildcomponentconfigs']; +- +-if ($options['help'] || (!$drop && !$install && !$buildconfig && !$buildcomponentconfigs && !$diag)) { ++$snapshot = $options['snapshot']; ++$restore = $options['restore']; ++$upgrade = $options['upgrade']; ++ ++if ( ++ $options['help'] || ( ++ !$drop && !$install && !$buildconfig && !$buildcomponentconfigs ++ && !$diag && !$upgrade && !$snapshot && !$restore ++ ) ++) { + $help = "Various PHPUnit utility functions + + Options: +@@ -114,6 +125,10 @@ Options: + --buildconfig Build /phpunit.xml from /phpunit.xml.dist that runs all tests + --buildcomponentconfigs + Build distributed phpunit.xml files for each component ++--upgrade Upgrade test site to latest version ++--snapshot Create snapshot of the current test site. Optionally specify name of the snapshot with ++ --snapshot=NAME. ++--restore=NAME Restore snapshot of test site with the specified name. + + -h, --help Print out this help + +@@ -155,4 +170,13 @@ if ($diag) { + } else if ($install) { + phpunit_util::install_site(); + exit(0); ++} else if ($upgrade) { ++ phpunit_util::upgrade_site(); ++ exit(0); ++} else if ($snapshot) { ++ phpunit_util::snapshot_site(is_string($snapshot) ? $snapshot : ''); ++ exit(0); ++} else if ($restore) { ++ phpunit_util::restore_site(is_string($restore) ? $restore : ''); ++ exit(0); + } +diff --git a/lib/phpunit/bootstrap.php b/lib/phpunit/bootstrap.php +index 9aedea38c3e..19ebfdec034 100644 +--- a/lib/phpunit/bootstrap.php ++++ b/lib/phpunit/bootstrap.php +@@ -151,7 +151,7 @@ if (!is_writable($CFG->phpunit_dataroot)) { + if (!file_exists("$CFG->phpunit_dataroot/phpunittestdir.txt")) { + if ($dh = opendir($CFG->phpunit_dataroot)) { + while (($file = readdir($dh)) !== false) { +- if ($file === 'phpunit' || $file === '.' || $file === '..' || $file === '.DS_Store') { ++ if ($file === 'phpunit' || $file === '.' || $file === '..' || $file === '.DS_Store' || str_ends_with($file, '.zip')) { + continue; + } + phpunit_bootstrap_error( +diff --git a/lib/phpunit/classes/util.php b/lib/phpunit/classes/util.php +index 86b52503aa3..efca15659f3 100644 +--- a/lib/phpunit/classes/util.php ++++ b/lib/phpunit/classes/util.php +@@ -451,6 +451,13 @@ class phpunit_util extends testing_util { + echo "Purging dataroot:\n"; + } + ++ // Preserve any snapshot zips so they survive the drop. ++ foreach (scandir(self::get_dataroot()) as $item) { ++ if (str_ends_with($item, '.zip')) { ++ static::$datarootskiponreset[] = $item; ++ } ++ } ++ + self::reset_dataroot(); + testing_initdataroot($CFG->dataroot, 'phpunit'); + +@@ -520,8 +527,373 @@ class phpunit_util extends testing_util { + } + + /** +- * Builds dirroot/phpunit.xml file using defaults from /phpunit.xml.dist +- * @static ++ * Create a snapshot of the current test site database and site data. ++ * ++ * @param string $snapshotname The name of the snapshot. ++ * @return void may terminate execution with exit code ++ */ ++ public static function snapshot_site(string $snapshotname) { ++ global $DB, $CFG; ++ ++ require($CFG->dirroot . '/version.php'); ++ ++ if (!self::is_test_site()) { ++ phpunit_bootstrap_error(PHPUNIT_EXITCODE_CONFIGERROR, 'Can not snapshot non-test site!!'); ++ } ++ ++ if (empty($DB->get_tables())) { ++ phpunit_bootstrap_error(PHPUNIT_EXITCODE_INSTALL, 'No database tables present, can not snapshot site!!'); ++ } ++ ++ $dataroot = self::get_dataroot(); ++ $dbtype = $CFG->dbtype; ++ $name = $snapshotname ? "$dbtype-$snapshotname-$version" : "$dbtype-snapshot-$version"; ++ $snapshotdir = $dataroot . "/$name"; ++ ++ // Clear old in-memory caches of snapshot data. ++ self::$tabledata = null; ++ self::$tablestructure = null; ++ self::$sequencenames = null; ++ ++ $sitedatapath = $dataroot . DIRECTORY_SEPARATOR . self::get_originaldatafilesjson(); ++ ++ if (file_exists($sitedatapath)) { ++ unlink($sitedatapath); ++ } ++ ++ self::reset_original_data(); ++ self::save_original_data_files(); ++ self::store_database_state(); ++ self::store_versions_hash(); ++ self::$lastdbwrites = $DB->perf_get_writes(); ++ ++ if (is_dir($snapshotdir)) { ++ remove_dir($snapshotdir); ++ } ++ ++ mkdir($snapshotdir); ++ ++ $filedir = $dataroot . DIRECTORY_SEPARATOR . 'filedir'; ++ ++ self::copy_filedir($filedir, $snapshotdir . DIRECTORY_SEPARATOR . 'filedir' . DIRECTORY_SEPARATOR); ++ ++ $datafiles = [ ++ "$dataroot" . DIRECTORY_SEPARATOR . "phpunit" . DIRECTORY_SEPARATOR . "tabledata.ser", ++ "$dataroot" . DIRECTORY_SEPARATOR . "phpunit" . DIRECTORY_SEPARATOR . "tablestructure.ser", ++ "$dataroot" . DIRECTORY_SEPARATOR . "phpunit" . DIRECTORY_SEPARATOR . "versionshash.txt", ++ "$dataroot" . DIRECTORY_SEPARATOR . "originaldatafiles.json", ++ ]; ++ ++ foreach ($datafiles as $file) { ++ if (!file_exists($file)) { ++ continue; ++ } ++ copy($file, $snapshotdir . DIRECTORY_SEPARATOR . basename($file)); ++ } ++ ++ $dirtozip = $dataroot . DIRECTORY_SEPARATOR . "$name"; ++ ++ $zip = new \ZipArchive(); ++ $zip->open("$dirtozip.zip", \ZipArchive::CREATE | \ZipArchive::OVERWRITE); ++ ++ $files = new \RecursiveIteratorIterator( ++ new \RecursiveDirectoryIterator($dirtozip, \RecursiveDirectoryIterator::SKIP_DOTS), ++ \RecursiveIteratorIterator::LEAVES_ONLY ++ ); ++ ++ foreach ($files as $file) { ++ if (!$file->isDir()) { ++ $filepath = $file->getRealPath(); ++ $filepathrelative = substr($filepath, strlen($dirtozip) + 1); ++ $zip->addFile($filepath, $filepathrelative); ++ } ++ } ++ ++ $zip->close(); ++ remove_dir($snapshotdir); ++ ++ echo "Snapshot successfully created in $dirtozip.zip\n"; ++ } ++ ++ /** ++ * Restore the database and dataroot to the state they were in when snapshot_site() was called. ++ * ++ * @param string $snapshot The snapshot to restore. ++ * @return void may terminate execution with exit code ++ */ ++ public static function restore_site(string $snapshot) { ++ global $DB, $CFG; ++ ++ require($CFG->dirroot . '/version.php'); ++ ++ if (!self::is_test_site()) { ++ phpunit_bootstrap_error(PHPUNIT_EXITCODE_CONFIGERROR, 'Can not restore non-test site!!'); ++ } ++ ++ if (!empty($DB->get_tables())) { ++ echo "Database tables are still present. Run php public/admin/tool/phpunit/cli/util.php --drop before restoring.\n"; ++ return; ++ } ++ ++ if (!$snapshot) { ++ echo "No snapshot name provided. Please provide the snapshot name as an argument.\n"; ++ return; ++ } ++ ++ $dataroot = self::get_dataroot(); ++ $zipfile = $dataroot . DIRECTORY_SEPARATOR . "$snapshot.zip"; ++ $destpath = $dataroot . DIRECTORY_SEPARATOR . "$snapshot"; ++ ++ $zip = new \ZipArchive(); ++ if ($zip->open($zipfile) === true) { ++ $zip->extractTo($destpath); ++ $zip->close(); ++ } else { ++ phpunit_bootstrap_error(PHPUNIT_EXITCODE_CONFIGERROR, "Can not open snapshot zip file $zipfile!!"); ++ } ++ ++ $snapshotdir = $dataroot . "/$snapshot"; ++ ++ // Map snapshot files to their active locations. ++ $datafiles = [ ++ "$snapshotdir" . DIRECTORY_SEPARATOR . "tabledata.ser" ++ => $dataroot . DIRECTORY_SEPARATOR . 'phpunit' . DIRECTORY_SEPARATOR . "tabledata.ser", ++ "$snapshotdir" . DIRECTORY_SEPARATOR . "tablestructure.ser" ++ => $dataroot . DIRECTORY_SEPARATOR . 'phpunit' . DIRECTORY_SEPARATOR . "tablestructure.ser", ++ "$snapshotdir" . DIRECTORY_SEPARATOR . "versionshash.txt" ++ => $dataroot . DIRECTORY_SEPARATOR . 'phpunit' . DIRECTORY_SEPARATOR . "versionshash.txt", ++ "$snapshotdir" . DIRECTORY_SEPARATOR . "originaldatafiles.json" ++ => $dataroot . DIRECTORY_SEPARATOR . self::get_originaldatafilesjson(), ++ ]; ++ ++ foreach ($datafiles as $src => $dest) { ++ if (!file_exists($src)) { ++ phpunit_bootstrap_error(PHPUNIT_EXITCODE_CONFIGERROR, "Required snapshot file $src does not exist!!"); ++ } ++ } ++ ++ self::$lastdbwrites = null; ++ self::$tabledata = null; ++ self::$tablestructure = null; ++ self::$sequencenames = null; ++ ++ echo "Restoring from $snapshot\n"; ++ ++ $phpunitdir = $dataroot . DIRECTORY_SEPARATOR . 'phpunit'; ++ if (!is_dir($phpunitdir)) { ++ mkdir($phpunitdir, 0777, true); ++ } ++ ++ // Copy snapshot DB control files to their active locations. ++ foreach ($datafiles as $src => $dest) { ++ copy($src, $dest); ++ } ++ ++ if (empty($DB->get_tables(false))) { ++ self::create_tables_from_snapshot($destpath); ++ } ++ ++ echo "Importing snapshot data...\n"; ++ // Restore the database to snapshot state. ++ self::reset_database(); ++ ++ echo "Data imported. Resetting datarooot...\n"; ++ ++ // Clear current dataroot before restoring filedir from snapshot. ++ static::$datarootskiponreset = ['.', '..', 'phpunittestdir.txt', 'phpunit', '.htaccess', $snapshot, "$snapshot.zip"]; ++ self::reset_dataroot(); ++ self::reset_original_data(); ++ ++ $snapshotfiledir = $snapshotdir . DIRECTORY_SEPARATOR . 'filedir'; ++ $filedir = $dataroot . DIRECTORY_SEPARATOR . 'filedir'; ++ ++ // Copy snapshot filedir back to dataroot. ++ self::copy_filedir($snapshotfiledir, $filedir); ++ ++ initialise_cfg(); ++ ++ // Execute all adhoc tasks. ++ while ($task = \core\task\manager::get_next_adhoc_task(time())) { ++ $task->execute(); ++ \core\task\manager::adhoc_task_complete($task); ++ } ++ ++ set_config('upgraderunning', 0); ++ ++ self::store_database_state(); ++ self::$lastdbwrites = $DB->perf_get_writes(); ++ ++ remove_dir($snapshotdir); ++ unlink($zipfile); ++ ++ echo "Snapshot successfully restored.\n"; ++ } ++ ++ /** ++ * Upgrade the test site and upgrade/install any new plugins that were added. ++ * ++ * @return void may terminate execution with exit code ++ */ ++ public static function upgrade_site() { ++ ++ if (!self::is_test_site()) { ++ phpunit_bootstrap_error(PHPUNIT_EXITCODE_CONFIGERROR, 'Can not upgrade non-test site!!'); ++ } ++ ++ if (self::is_test_data_updated()) { ++ echo "No database changes detected, skipping upgrade.\n"; ++ return; ++ } ++ ++ initialise_cfg(); ++ upgrade_noncore(true); ++ set_config('upgraderunning', 0); ++ self::store_versions_hash(); ++ self::store_database_state(); ++ } ++ ++ /** ++ * Helper method that copies all contents from one directory to another, creating directories as needed. ++ * ++ * @param string $sourcedir the source directory to copy ++ * @param string $dirtocreate the destination directory to create and copy files to ++ * @return void ++ */ ++ private static function copy_filedir($sourcedir, $dirtocreate) { ++ ++ if (!file_exists($sourcedir)) { ++ echo "Source directory $sourcedir does not exist!\n"; ++ return; ++ } ++ ++ $iterator = new \RecursiveIteratorIterator( ++ new \RecursiveDirectoryIterator($sourcedir, \RecursiveDirectoryIterator::SKIP_DOTS), ++ \RecursiveIteratorIterator::SELF_FIRST ++ ); ++ ++ if (!is_dir($dirtocreate)) { ++ mkdir($dirtocreate, 0777, true); ++ } ++ ++ foreach ($iterator as $item) { ++ $destpath = $dirtocreate . DIRECTORY_SEPARATOR . $iterator->getSubPathName(); ++ if ($item->isDir()) { ++ if (!is_dir($destpath)) { ++ mkdir($destpath, 0777, true); ++ } ++ } else { ++ copy($item->getPathname(), $destpath); ++ } ++ } ++ } ++ ++ /** ++ * Create the database tables based on the stored snapshot structure. ++ * ++ * @param string $snapshotdir the target snapshot directory to create the tables from. ++ * @param bool $showprogress whether to show progress during table creation. ++ * @return void may terminate execution with exit code ++ */ ++ private static function create_tables_from_snapshot(string $snapshotdir, bool $showprogress = true) { ++ global $DB; ++ ++ $dbman = $DB->get_manager(); ++ ++ $structurefile = $snapshotdir . DIRECTORY_SEPARATOR . 'tablestructure.ser'; ++ if (!file_exists($structurefile)) { ++ phpunit_bootstrap_error(PHPUNIT_EXITCODE_CONFIGERROR, "Snapshot file $structurefile does not exist!!"); ++ } ++ $tablestructure = unserialize(file_get_contents($structurefile)); ++ if (!is_array($tablestructure)) { ++ phpunit_bootstrap_error(PHPUNIT_EXITCODE_CONFIGERROR, "Invalid tablestructure.ser in snapshot $snapshotdir"); ++ } ++ ++ $dotsonline = 0; ++ foreach ($tablestructure as $tablename => $columns) { ++ // If the table already exists, skip it. ++ if ($dbman->table_exists($tablename)) { ++ continue; ++ } ++ ++ $table = new \xmldb_table($tablename); ++ ++ // Build an xmldb_field for every column. ++ foreach ($columns as $colname => $col) { ++ $field = new \xmldb_field($colname); ++ $sequence = false; ++ // Map each column meta type to the appropriate xmldb type. ++ switch ($col->meta_type) { ++ case 'R': ++ $field->setType(XMLDB_TYPE_INTEGER); ++ $field->setLength(10); ++ $sequence = true; ++ break; ++ case 'I': ++ $field->setType(XMLDB_TYPE_INTEGER); ++ $field->setLength($col->max_length ?: 10); ++ break; ++ case 'N': ++ $floattypes = ['float', 'double', 'float4', 'float8', 'double precision']; ++ if (in_array(strtolower($col->type ?? ''), $floattypes)) { ++ $field->setType(XMLDB_TYPE_FLOAT); ++ } else { ++ $field->setType(XMLDB_TYPE_NUMBER); ++ $field->setLength($col->max_length ?: 10); ++ $field->setDecimals($col->scale ?: 0); ++ } ++ break; ++ case 'C': ++ $field->setType(XMLDB_TYPE_CHAR); ++ $field->setLength($col->max_length ?: 255); ++ break; ++ case 'X': ++ $field->setType(XMLDB_TYPE_TEXT); ++ break; ++ case 'B': ++ $field->setType(XMLDB_TYPE_BINARY); ++ break; ++ case 'L': ++ $field->setType(XMLDB_TYPE_INTEGER); ++ $field->setLength(1); ++ break; ++ default: ++ $field->setType(XMLDB_TYPE_INTEGER); ++ $field->setLength($col->max_length ?: 10); ++ } ++ $field->setNotNull($col->not_null); ++ if ($col->has_default && $col->default_value !== '') { ++ $field->setDefault($col->default_value); ++ } ++ if ($sequence) { ++ $field->setSequence(true); ++ } ++ $table->addField($field); ++ } ++ if (isset($columns['id'])) { ++ // Add a primary key for the 'id' column. ++ $key = new \xmldb_key('primary', XMLDB_KEY_PRIMARY, ['id']); ++ $table->addKey($key); ++ } ++ $dbman->create_table($table); ++ ++ if ($dotsonline == 60) { ++ if ($showprogress) { ++ echo "\n"; ++ } ++ $dotsonline = 0; ++ } ++ if ($showprogress) { ++ echo '.'; ++ } ++ $dotsonline += 1; ++ } ++ ++ echo "\nTables successfully created from snapshot.\n"; ++ } ++ ++ /** ++ * Builds root/phpunit.xml file using defaults from /phpunit.xml.dist ++ * + * @return bool true means main config file created, false means only dataroot file created + */ + public static function build_config_file() { +diff --git a/lib/testing/classes/util.php b/lib/testing/classes/util.php +index 5315c77c94b..dea37ca6a56 100644 +--- a/lib/testing/classes/util.php ++++ b/lib/testing/classes/util.php +@@ -125,6 +125,13 @@ abstract class testing_util { + return substr($classname, 0, strpos($classname, '_')); + } + ++ /** ++ * Reset the original data. ++ */ ++ protected static function reset_original_data() { ++ self::$originaldatafilesjsonadded = false; ++ } ++ + /** + * Get data generator + * @static +-- +2.43.0 + diff --git a/.github/patches/501-501-phpunit-restore.patch b/.github/patches/501-501-phpunit-restore.patch new file mode 100644 index 00000000..d781c3f9 --- /dev/null +++ b/.github/patches/501-501-phpunit-restore.patch @@ -0,0 +1,494 @@ +From 6bdbe047e859f83718c56333449f1ce2ca4e602a Mon Sep 17 00:00:00 2001 +From: Abhinav Gandham +Date: Tue, 26 May 2026 17:26:32 +1000 +Subject: [PATCH] MDL-88495 Unit tests: Snapshot and restore for test site. + +--- + public/admin/tool/phpunit/cli/util.php | 28 +- + public/lib/phpunit/bootstrap.php | 2 +- + public/lib/phpunit/classes/util.php | 372 +++++++++++++++++++++++++ + public/lib/testing/classes/util.php | 7 + + 4 files changed, 406 insertions(+), 3 deletions(-) + +diff --git a/public/admin/tool/phpunit/cli/util.php b/public/admin/tool/phpunit/cli/util.php +index ff5eb956ddd..2ac4af73b74 100644 +--- a/public/admin/tool/phpunit/cli/util.php ++++ b/public/admin/tool/phpunit/cli/util.php +@@ -47,6 +47,9 @@ list($options, $unrecognized) = cli_get_params( + 'buildcomponentconfigs' => false, + 'diag' => false, + 'run' => false, ++ 'upgrade' => false, ++ 'snapshot' => '', ++ 'restore' => '', + 'help' => false, + ], + [ +@@ -102,8 +105,16 @@ $drop = $options['drop']; + $install = $options['install']; + $buildconfig = $options['buildconfig']; + $buildcomponentconfigs = $options['buildcomponentconfigs']; +- +-if ($options['help'] || (!$drop && !$install && !$buildconfig && !$buildcomponentconfigs && !$diag)) { ++$snapshot = $options['snapshot']; ++$restore = $options['restore']; ++$upgrade = $options['upgrade']; ++ ++if ( ++ $options['help'] || ( ++ !$drop && !$install && !$buildconfig && !$buildcomponentconfigs ++ && !$diag && !$upgrade && !$snapshot && !$restore ++ ) ++) { + $help = "Various PHPUnit utility functions + + Options: +@@ -114,6 +125,10 @@ Options: + --buildconfig Build /phpunit.xml from /phpunit.xml.dist that runs all tests + --buildcomponentconfigs + Build distributed phpunit.xml files for each component ++--upgrade Upgrade test site to latest version ++--snapshot Create snapshot of the current test site. Optionally specify name of the snapshot with ++ --snapshot=NAME. ++--restore=NAME Restore snapshot of test site with the specified name. + + -h, --help Print out this help + +@@ -155,4 +170,13 @@ if ($diag) { + } else if ($install) { + phpunit_util::install_site(); + exit(0); ++} else if ($upgrade) { ++ phpunit_util::upgrade_site(); ++ exit(0); ++} else if ($snapshot) { ++ phpunit_util::snapshot_site(is_string($snapshot) ? $snapshot : ''); ++ exit(0); ++} else if ($restore) { ++ phpunit_util::restore_site(is_string($restore) ? $restore : ''); ++ exit(0); + } +diff --git a/public/lib/phpunit/bootstrap.php b/public/lib/phpunit/bootstrap.php +index 8916ad69765..60da5faee50 100644 +--- a/public/lib/phpunit/bootstrap.php ++++ b/public/lib/phpunit/bootstrap.php +@@ -151,7 +151,7 @@ if (!is_writable($CFG->phpunit_dataroot)) { + if (!file_exists("$CFG->phpunit_dataroot/phpunittestdir.txt")) { + if ($dh = opendir($CFG->phpunit_dataroot)) { + while (($file = readdir($dh)) !== false) { +- if ($file === 'phpunit' || $file === '.' || $file === '..' || $file === '.DS_Store') { ++ if ($file === 'phpunit' || $file === '.' || $file === '..' || $file === '.DS_Store' || str_ends_with($file, '.zip')) { + continue; + } + phpunit_bootstrap_error( +diff --git a/public/lib/phpunit/classes/util.php b/public/lib/phpunit/classes/util.php +index fcc574117ae..7436abae5f6 100644 +--- a/public/lib/phpunit/classes/util.php ++++ b/public/lib/phpunit/classes/util.php +@@ -454,6 +454,13 @@ class phpunit_util extends testing_util { + echo "Purging dataroot:\n"; + } + ++ // Preserve any snapshot zips so they survive the drop. ++ foreach (scandir(self::get_dataroot()) as $item) { ++ if (str_ends_with($item, '.zip')) { ++ static::$datarootskiponreset[] = $item; ++ } ++ } ++ + self::reset_dataroot(); + testing_initdataroot($CFG->dataroot, 'phpunit'); + +@@ -524,6 +531,371 @@ class phpunit_util extends testing_util { + self::store_database_state(); + } + ++ /** ++ * Create a snapshot of the current test site database and site data. ++ * ++ * @param string $snapshotname The name of the snapshot. ++ * @return void may terminate execution with exit code ++ */ ++ public static function snapshot_site(string $snapshotname) { ++ global $DB, $CFG; ++ ++ require($CFG->dirroot . '/version.php'); ++ ++ if (!self::is_test_site()) { ++ phpunit_bootstrap_error(PHPUNIT_EXITCODE_CONFIGERROR, 'Can not snapshot non-test site!!'); ++ } ++ ++ if (empty($DB->get_tables())) { ++ phpunit_bootstrap_error(PHPUNIT_EXITCODE_INSTALL, 'No database tables present, can not snapshot site!!'); ++ } ++ ++ $dataroot = self::get_dataroot(); ++ $dbtype = $CFG->dbtype; ++ $name = $snapshotname ? "$dbtype-$snapshotname-$version" : "$dbtype-snapshot-$version"; ++ $snapshotdir = $dataroot . "/$name"; ++ ++ // Clear old in-memory caches of snapshot data. ++ self::$tabledata = null; ++ self::$tablestructure = null; ++ self::$sequencenames = null; ++ ++ $sitedatapath = $dataroot . DIRECTORY_SEPARATOR . self::get_originaldatafilesjson(); ++ ++ if (file_exists($sitedatapath)) { ++ unlink($sitedatapath); ++ } ++ ++ self::reset_original_data(); ++ self::save_original_data_files(); ++ self::store_database_state(); ++ self::store_versions_hash(); ++ self::$lastdbwrites = $DB->perf_get_writes(); ++ ++ if (is_dir($snapshotdir)) { ++ remove_dir($snapshotdir); ++ } ++ ++ mkdir($snapshotdir); ++ ++ $filedir = $dataroot . DIRECTORY_SEPARATOR . 'filedir'; ++ ++ self::copy_filedir($filedir, $snapshotdir . DIRECTORY_SEPARATOR . 'filedir' . DIRECTORY_SEPARATOR); ++ ++ $datafiles = [ ++ "$dataroot" . DIRECTORY_SEPARATOR . "phpunit" . DIRECTORY_SEPARATOR . "tabledata.ser", ++ "$dataroot" . DIRECTORY_SEPARATOR . "phpunit" . DIRECTORY_SEPARATOR . "tablestructure.ser", ++ "$dataroot" . DIRECTORY_SEPARATOR . "phpunit" . DIRECTORY_SEPARATOR . "versionshash.txt", ++ "$dataroot" . DIRECTORY_SEPARATOR . "originaldatafiles.json", ++ ]; ++ ++ foreach ($datafiles as $file) { ++ if (!file_exists($file)) { ++ continue; ++ } ++ copy($file, $snapshotdir . DIRECTORY_SEPARATOR . basename($file)); ++ } ++ ++ $dirtozip = $dataroot . DIRECTORY_SEPARATOR . "$name"; ++ ++ $zip = new \ZipArchive(); ++ $zip->open("$dirtozip.zip", \ZipArchive::CREATE | \ZipArchive::OVERWRITE); ++ ++ $files = new \RecursiveIteratorIterator( ++ new \RecursiveDirectoryIterator($dirtozip, \RecursiveDirectoryIterator::SKIP_DOTS), ++ \RecursiveIteratorIterator::LEAVES_ONLY ++ ); ++ ++ foreach ($files as $file) { ++ if (!$file->isDir()) { ++ $filepath = $file->getRealPath(); ++ $filepathrelative = substr($filepath, strlen($dirtozip) + 1); ++ $zip->addFile($filepath, $filepathrelative); ++ } ++ } ++ ++ $zip->close(); ++ remove_dir($snapshotdir); ++ ++ echo "Snapshot successfully created in $dirtozip.zip\n"; ++ } ++ ++ /** ++ * Restore the database and dataroot to the state they were in when snapshot_site() was called. ++ * ++ * @param string $snapshot The snapshot to restore. ++ * @return void may terminate execution with exit code ++ */ ++ public static function restore_site(string $snapshot) { ++ global $DB, $CFG; ++ ++ require($CFG->dirroot . '/version.php'); ++ ++ if (!self::is_test_site()) { ++ phpunit_bootstrap_error(PHPUNIT_EXITCODE_CONFIGERROR, 'Can not restore non-test site!!'); ++ } ++ ++ if (!empty($DB->get_tables())) { ++ echo "Database tables are still present. Run php public/admin/tool/phpunit/cli/util.php --drop before restoring.\n"; ++ return; ++ } ++ ++ if (!$snapshot) { ++ echo "No snapshot name provided. Please provide the snapshot name as an argument.\n"; ++ return; ++ } ++ ++ $dataroot = self::get_dataroot(); ++ $zipfile = $dataroot . DIRECTORY_SEPARATOR . "$snapshot.zip"; ++ $destpath = $dataroot . DIRECTORY_SEPARATOR . "$snapshot"; ++ ++ $zip = new \ZipArchive(); ++ if ($zip->open($zipfile) === true) { ++ $zip->extractTo($destpath); ++ $zip->close(); ++ } else { ++ phpunit_bootstrap_error(PHPUNIT_EXITCODE_CONFIGERROR, "Can not open snapshot zip file $zipfile!!"); ++ } ++ ++ $snapshotdir = $dataroot . "/$snapshot"; ++ ++ // Map snapshot files to their active locations. ++ $datafiles = [ ++ "$snapshotdir" . DIRECTORY_SEPARATOR . "tabledata.ser" ++ => $dataroot . DIRECTORY_SEPARATOR . 'phpunit' . DIRECTORY_SEPARATOR . "tabledata.ser", ++ "$snapshotdir" . DIRECTORY_SEPARATOR . "tablestructure.ser" ++ => $dataroot . DIRECTORY_SEPARATOR . 'phpunit' . DIRECTORY_SEPARATOR . "tablestructure.ser", ++ "$snapshotdir" . DIRECTORY_SEPARATOR . "versionshash.txt" ++ => $dataroot . DIRECTORY_SEPARATOR . 'phpunit' . DIRECTORY_SEPARATOR . "versionshash.txt", ++ "$snapshotdir" . DIRECTORY_SEPARATOR . "originaldatafiles.json" ++ => $dataroot . DIRECTORY_SEPARATOR . self::get_originaldatafilesjson(), ++ ]; ++ ++ foreach ($datafiles as $src => $dest) { ++ if (!file_exists($src)) { ++ phpunit_bootstrap_error(PHPUNIT_EXITCODE_CONFIGERROR, "Required snapshot file $src does not exist!!"); ++ } ++ } ++ ++ self::$lastdbwrites = null; ++ self::$tabledata = null; ++ self::$tablestructure = null; ++ self::$sequencenames = null; ++ ++ echo "Restoring from $snapshot\n"; ++ ++ $phpunitdir = $dataroot . DIRECTORY_SEPARATOR . 'phpunit'; ++ if (!is_dir($phpunitdir)) { ++ mkdir($phpunitdir, 0777, true); ++ } ++ ++ // Copy snapshot DB control files to their active locations. ++ foreach ($datafiles as $src => $dest) { ++ copy($src, $dest); ++ } ++ ++ if (empty($DB->get_tables(false))) { ++ self::create_tables_from_snapshot($destpath); ++ } ++ ++ echo "Importing snapshot data...\n"; ++ // Restore the database to snapshot state. ++ self::reset_database(); ++ ++ echo "Data imported. Resetting datarooot...\n"; ++ ++ // Clear current dataroot before restoring filedir from snapshot. ++ static::$datarootskiponreset = ['.', '..', 'phpunittestdir.txt', 'phpunit', '.htaccess', $snapshot, "$snapshot.zip"]; ++ self::reset_dataroot(); ++ self::reset_original_data(); ++ ++ $snapshotfiledir = $snapshotdir . DIRECTORY_SEPARATOR . 'filedir'; ++ $filedir = $dataroot . DIRECTORY_SEPARATOR . 'filedir'; ++ ++ // Copy snapshot filedir back to dataroot. ++ self::copy_filedir($snapshotfiledir, $filedir); ++ ++ initialise_cfg(); ++ ++ // Execute all adhoc tasks. ++ while ($task = \core\task\manager::get_next_adhoc_task(time())) { ++ $task->execute(); ++ \core\task\manager::adhoc_task_complete($task); ++ } ++ ++ set_config('upgraderunning', 0); ++ ++ self::store_database_state(); ++ self::$lastdbwrites = $DB->perf_get_writes(); ++ ++ remove_dir($snapshotdir); ++ unlink($zipfile); ++ ++ echo "Snapshot successfully restored.\n"; ++ } ++ ++ /** ++ * Upgrade the test site and upgrade/install any new plugins that were added. ++ * ++ * @return void may terminate execution with exit code ++ */ ++ public static function upgrade_site() { ++ ++ if (!self::is_test_site()) { ++ phpunit_bootstrap_error(PHPUNIT_EXITCODE_CONFIGERROR, 'Can not upgrade non-test site!!'); ++ } ++ ++ if (self::is_test_data_updated()) { ++ echo "No database changes detected, skipping upgrade.\n"; ++ return; ++ } ++ ++ initialise_cfg(); ++ upgrade_noncore(true); ++ set_config('upgraderunning', 0); ++ self::store_versions_hash(); ++ self::store_database_state(); ++ } ++ ++ /** ++ * Helper method that copies all contents from one directory to another, creating directories as needed. ++ * ++ * @param string $sourcedir the source directory to copy ++ * @param string $dirtocreate the destination directory to create and copy files to ++ * @return void ++ */ ++ private static function copy_filedir($sourcedir, $dirtocreate) { ++ ++ if (!file_exists($sourcedir)) { ++ echo "Source directory $sourcedir does not exist!\n"; ++ return; ++ } ++ ++ $iterator = new \RecursiveIteratorIterator( ++ new \RecursiveDirectoryIterator($sourcedir, \RecursiveDirectoryIterator::SKIP_DOTS), ++ \RecursiveIteratorIterator::SELF_FIRST ++ ); ++ ++ if (!is_dir($dirtocreate)) { ++ mkdir($dirtocreate, 0777, true); ++ } ++ ++ foreach ($iterator as $item) { ++ $destpath = $dirtocreate . DIRECTORY_SEPARATOR . $iterator->getSubPathName(); ++ if ($item->isDir()) { ++ if (!is_dir($destpath)) { ++ mkdir($destpath, 0777, true); ++ } ++ } else { ++ copy($item->getPathname(), $destpath); ++ } ++ } ++ } ++ ++ /** ++ * Create the database tables based on the stored snapshot structure. ++ * ++ * @param string $snapshotdir the target snapshot directory to create the tables from. ++ * @param bool $showprogress whether to show progress during table creation. ++ * @return void may terminate execution with exit code ++ */ ++ private static function create_tables_from_snapshot(string $snapshotdir, bool $showprogress = true) { ++ global $DB; ++ ++ $dbman = $DB->get_manager(); ++ ++ $structurefile = $snapshotdir . DIRECTORY_SEPARATOR . 'tablestructure.ser'; ++ if (!file_exists($structurefile)) { ++ phpunit_bootstrap_error(PHPUNIT_EXITCODE_CONFIGERROR, "Snapshot file $structurefile does not exist!!"); ++ } ++ $tablestructure = unserialize(file_get_contents($structurefile)); ++ if (!is_array($tablestructure)) { ++ phpunit_bootstrap_error(PHPUNIT_EXITCODE_CONFIGERROR, "Invalid tablestructure.ser in snapshot $snapshotdir"); ++ } ++ ++ $dotsonline = 0; ++ foreach ($tablestructure as $tablename => $columns) { ++ // If the table already exists, skip it. ++ if ($dbman->table_exists($tablename)) { ++ continue; ++ } ++ ++ $table = new \xmldb_table($tablename); ++ ++ // Build an xmldb_field for every column. ++ foreach ($columns as $colname => $col) { ++ $field = new \xmldb_field($colname); ++ $sequence = false; ++ // Map each column meta type to the appropriate xmldb type. ++ switch ($col->meta_type) { ++ case 'R': ++ $field->setType(XMLDB_TYPE_INTEGER); ++ $field->setLength(10); ++ $sequence = true; ++ break; ++ case 'I': ++ $field->setType(XMLDB_TYPE_INTEGER); ++ $field->setLength($col->max_length ?: 10); ++ break; ++ case 'N': ++ $floattypes = ['float', 'double', 'float4', 'float8', 'double precision']; ++ if (in_array(strtolower($col->type ?? ''), $floattypes)) { ++ $field->setType(XMLDB_TYPE_FLOAT); ++ } else { ++ $field->setType(XMLDB_TYPE_NUMBER); ++ $field->setLength($col->max_length ?: 10); ++ $field->setDecimals($col->scale ?: 0); ++ } ++ break; ++ case 'C': ++ $field->setType(XMLDB_TYPE_CHAR); ++ $field->setLength($col->max_length ?: 255); ++ break; ++ case 'X': ++ $field->setType(XMLDB_TYPE_TEXT); ++ break; ++ case 'B': ++ $field->setType(XMLDB_TYPE_BINARY); ++ break; ++ case 'L': ++ $field->setType(XMLDB_TYPE_INTEGER); ++ $field->setLength(1); ++ break; ++ default: ++ $field->setType(XMLDB_TYPE_INTEGER); ++ $field->setLength($col->max_length ?: 10); ++ } ++ $field->setNotNull($col->not_null); ++ if ($col->has_default && $col->default_value !== '') { ++ $field->setDefault($col->default_value); ++ } ++ if ($sequence) { ++ $field->setSequence(true); ++ } ++ $table->addField($field); ++ } ++ if (isset($columns['id'])) { ++ // Add a primary key for the 'id' column. ++ $key = new \xmldb_key('primary', XMLDB_KEY_PRIMARY, ['id']); ++ $table->addKey($key); ++ } ++ $dbman->create_table($table); ++ ++ if ($dotsonline == 60) { ++ if ($showprogress) { ++ echo "\n"; ++ } ++ $dotsonline = 0; ++ } ++ if ($showprogress) { ++ echo '.'; ++ } ++ $dotsonline += 1; ++ } ++ ++ echo "\nTables successfully created from snapshot.\n"; ++ } ++ + /** + * Builds root/phpunit.xml file using defaults from /phpunit.xml.dist + * +diff --git a/public/lib/testing/classes/util.php b/public/lib/testing/classes/util.php +index 3073cde3bf7..070adfa78ce 100644 +--- a/public/lib/testing/classes/util.php ++++ b/public/lib/testing/classes/util.php +@@ -125,6 +125,13 @@ abstract class testing_util { + return substr($classname, 0, strpos($classname, '_')); + } + ++ /** ++ * Reset the original data. ++ */ ++ protected static function reset_original_data() { ++ self::$originaldatafilesjsonadded = false; ++ } ++ + /** + * Get data generator + * @static +-- +2.43.0 + diff --git a/.github/patches/502-999-phpunit-restore.patch b/.github/patches/502-999-phpunit-restore.patch new file mode 100644 index 00000000..3f3e5ddc --- /dev/null +++ b/.github/patches/502-999-phpunit-restore.patch @@ -0,0 +1,494 @@ +From 3f43ee764e2582fb0d32f6e710fccea61c1e7e5a Mon Sep 17 00:00:00 2001 +From: Abhinav Gandham +Date: Fri, 22 May 2026 15:05:47 +1000 +Subject: [PATCH] MDL-88495 Unit tests: Snapshot and restore for test site. + +--- + public/admin/tool/phpunit/cli/util.php | 28 +- + .../lib/classes/test/phpunit/phpunit_util.php | 372 ++++++++++++++++++ + public/lib/classes/test/testing_util.php | 7 + + public/lib/phpunit/bootstrap.php | 2 +- + 4 files changed, 406 insertions(+), 3 deletions(-) + +diff --git a/public/admin/tool/phpunit/cli/util.php b/public/admin/tool/phpunit/cli/util.php +index 0137344ba03..f1aa8a15290 100644 +--- a/public/admin/tool/phpunit/cli/util.php ++++ b/public/admin/tool/phpunit/cli/util.php +@@ -49,6 +49,9 @@ list($options, $unrecognized) = cli_get_params( + 'buildcomponentconfigs' => false, + 'diag' => false, + 'run' => false, ++ 'upgrade' => false, ++ 'snapshot' => '', ++ 'restore' => '', + 'help' => false, + ], + [ +@@ -115,8 +118,16 @@ $drop = $options['drop']; + $install = $options['install']; + $buildconfig = $options['buildconfig']; + $buildcomponentconfigs = $options['buildcomponentconfigs']; +- +-if ($options['help'] || (!$drop && !$install && !$buildconfig && !$buildcomponentconfigs && !$diag)) { ++$snapshot = $options['snapshot']; ++$restore = $options['restore']; ++$upgrade = $options['upgrade']; ++ ++if ( ++ $options['help'] || ( ++ !$drop && !$install && !$buildconfig && !$buildcomponentconfigs ++ && !$diag && !$upgrade && !$snapshot && !$restore ++ ) ++) { + $help = "Various PHPUnit utility functions + + Options: +@@ -127,6 +138,10 @@ Options: + --buildconfig Build /phpunit.xml from /phpunit.xml.dist that runs all tests + --buildcomponentconfigs + Build distributed phpunit.xml files for each component ++--upgrade Upgrade test site to latest version ++--snapshot Create snapshot of the current test site. Optionally specify name of the snapshot with ++ --snapshot=NAME. ++--restore=NAME Restore snapshot of test site with the specified name. + + -h, --help Print out this help + +@@ -168,4 +183,13 @@ if ($diag) { + } else if ($install) { + phpunit_util::install_site(); + exit(0); ++} else if ($upgrade) { ++ phpunit_util::upgrade_site(); ++ exit(0); ++} else if ($snapshot) { ++ phpunit_util::snapshot_site(is_string($snapshot) ? $snapshot : ''); ++ exit(0); ++} else if ($restore) { ++ phpunit_util::restore_site(is_string($restore) ? $restore : ''); ++ exit(0); + } +diff --git a/public/lib/classes/test/phpunit/phpunit_util.php b/public/lib/classes/test/phpunit/phpunit_util.php +index adfd5a61c2c..d9675291181 100644 +--- a/public/lib/classes/test/phpunit/phpunit_util.php ++++ b/public/lib/classes/test/phpunit/phpunit_util.php +@@ -436,6 +436,13 @@ class phpunit_util extends \core\test\testing_util { + echo "Purging dataroot:\n"; + } + ++ // Preserve any snapshot zips so they survive the drop. ++ foreach (scandir(self::get_dataroot()) as $item) { ++ if (str_ends_with($item, '.zip')) { ++ static::$datarootskiponreset[] = $item; ++ } ++ } ++ + self::reset_dataroot(); + testing_initdataroot($CFG->dataroot, 'phpunit'); + +@@ -506,6 +513,371 @@ class phpunit_util extends \core\test\testing_util { + self::store_database_state(); + } + ++ /** ++ * Create a snapshot of the current test site database and site data. ++ * ++ * @param string $snapshotname The name of the snapshot. ++ * @return void may terminate execution with exit code ++ */ ++ public static function snapshot_site(string $snapshotname) { ++ global $DB, $CFG; ++ ++ require($CFG->dirroot . '/version.php'); ++ ++ if (!self::is_test_site()) { ++ phpunit_bootstrap_error(PHPUNIT_EXITCODE_CONFIGERROR, 'Can not snapshot non-test site!!'); ++ } ++ ++ if (empty($DB->get_tables())) { ++ phpunit_bootstrap_error(PHPUNIT_EXITCODE_INSTALL, 'No database tables present, can not snapshot site!!'); ++ } ++ ++ $dataroot = self::get_dataroot(); ++ $dbtype = $CFG->dbtype; ++ $name = $snapshotname ? "$dbtype-$snapshotname-$version" : "$dbtype-snapshot-$version"; ++ $snapshotdir = $dataroot . "/$name"; ++ ++ // Clear old in-memory caches of snapshot data. ++ self::$tabledata = null; ++ self::$tablestructure = null; ++ self::$sequencenames = null; ++ ++ $sitedatapath = $dataroot . DIRECTORY_SEPARATOR . self::get_originaldatafilesjson(); ++ ++ if (file_exists($sitedatapath)) { ++ unlink($sitedatapath); ++ } ++ ++ self::reset_original_data(); ++ self::save_original_data_files(); ++ self::store_database_state(); ++ self::store_versions_hash(); ++ self::$lastdbwrites = $DB->perf_get_writes(); ++ ++ if (is_dir($snapshotdir)) { ++ remove_dir($snapshotdir); ++ } ++ ++ mkdir($snapshotdir); ++ ++ $filedir = $dataroot . DIRECTORY_SEPARATOR . 'filedir'; ++ ++ self::copy_filedir($filedir, $snapshotdir . DIRECTORY_SEPARATOR . 'filedir' . DIRECTORY_SEPARATOR); ++ ++ $datafiles = [ ++ "$dataroot" . DIRECTORY_SEPARATOR . "phpunit" . DIRECTORY_SEPARATOR . "tabledata.ser", ++ "$dataroot" . DIRECTORY_SEPARATOR . "phpunit" . DIRECTORY_SEPARATOR . "tablestructure.ser", ++ "$dataroot" . DIRECTORY_SEPARATOR . "phpunit" . DIRECTORY_SEPARATOR . "versionshash.txt", ++ "$dataroot" . DIRECTORY_SEPARATOR . "originaldatafiles.json", ++ ]; ++ ++ foreach ($datafiles as $file) { ++ if (!file_exists($file)) { ++ continue; ++ } ++ copy($file, $snapshotdir . DIRECTORY_SEPARATOR . basename($file)); ++ } ++ ++ $dirtozip = $dataroot . DIRECTORY_SEPARATOR . "$name"; ++ ++ $zip = new \ZipArchive(); ++ $zip->open("$dirtozip.zip", \ZipArchive::CREATE | \ZipArchive::OVERWRITE); ++ ++ $files = new \RecursiveIteratorIterator( ++ new \RecursiveDirectoryIterator($dirtozip, \RecursiveDirectoryIterator::SKIP_DOTS), ++ \RecursiveIteratorIterator::LEAVES_ONLY ++ ); ++ ++ foreach ($files as $file) { ++ if (!$file->isDir()) { ++ $filepath = $file->getRealPath(); ++ $filepathrelative = substr($filepath, strlen($dirtozip) + 1); ++ $zip->addFile($filepath, $filepathrelative); ++ } ++ } ++ ++ $zip->close(); ++ remove_dir($snapshotdir); ++ ++ echo "Snapshot successfully created in $dirtozip.zip\n"; ++ } ++ ++ /** ++ * Restore the database and dataroot to the state they were in when snapshot_site() was called. ++ * ++ * @param string $snapshot The snapshot to restore. ++ * @return void may terminate execution with exit code ++ */ ++ public static function restore_site(string $snapshot) { ++ global $DB, $CFG; ++ ++ require($CFG->dirroot . '/version.php'); ++ ++ if (!self::is_test_site()) { ++ phpunit_bootstrap_error(PHPUNIT_EXITCODE_CONFIGERROR, 'Can not restore non-test site!!'); ++ } ++ ++ if (!empty($DB->get_tables())) { ++ echo "Database tables are still present. Run php public/admin/tool/phpunit/cli/util.php --drop before restoring.\n"; ++ return; ++ } ++ ++ if (!$snapshot) { ++ echo "No snapshot name provided. Please provide the snapshot name as an argument.\n"; ++ return; ++ } ++ ++ $dataroot = self::get_dataroot(); ++ $zipfile = $dataroot . DIRECTORY_SEPARATOR . "$snapshot.zip"; ++ $destpath = $dataroot . DIRECTORY_SEPARATOR . "$snapshot"; ++ ++ $zip = new \ZipArchive(); ++ if ($zip->open($zipfile) === true) { ++ $zip->extractTo($destpath); ++ $zip->close(); ++ } else { ++ phpunit_bootstrap_error(PHPUNIT_EXITCODE_CONFIGERROR, "Can not open snapshot zip file $zipfile!!"); ++ } ++ ++ $snapshotdir = $dataroot . "/$snapshot"; ++ ++ // Map snapshot files to their active locations. ++ $datafiles = [ ++ "$snapshotdir" . DIRECTORY_SEPARATOR . "tabledata.ser" ++ => $dataroot . DIRECTORY_SEPARATOR . 'phpunit' . DIRECTORY_SEPARATOR . "tabledata.ser", ++ "$snapshotdir" . DIRECTORY_SEPARATOR . "tablestructure.ser" ++ => $dataroot . DIRECTORY_SEPARATOR . 'phpunit' . DIRECTORY_SEPARATOR . "tablestructure.ser", ++ "$snapshotdir" . DIRECTORY_SEPARATOR . "versionshash.txt" ++ => $dataroot . DIRECTORY_SEPARATOR . 'phpunit' . DIRECTORY_SEPARATOR . "versionshash.txt", ++ "$snapshotdir" . DIRECTORY_SEPARATOR . "originaldatafiles.json" ++ => $dataroot . DIRECTORY_SEPARATOR . self::get_originaldatafilesjson(), ++ ]; ++ ++ foreach ($datafiles as $src => $dest) { ++ if (!file_exists($src)) { ++ phpunit_bootstrap_error(PHPUNIT_EXITCODE_CONFIGERROR, "Required snapshot file $src does not exist!!"); ++ } ++ } ++ ++ self::$lastdbwrites = null; ++ self::$tabledata = null; ++ self::$tablestructure = null; ++ self::$sequencenames = null; ++ ++ echo "Restoring from $snapshot\n"; ++ ++ $phpunitdir = $dataroot . DIRECTORY_SEPARATOR . 'phpunit'; ++ if (!is_dir($phpunitdir)) { ++ mkdir($phpunitdir, 0777, true); ++ } ++ ++ // Copy snapshot DB control files to their active locations. ++ foreach ($datafiles as $src => $dest) { ++ copy($src, $dest); ++ } ++ ++ if (empty($DB->get_tables(false))) { ++ self::create_tables_from_snapshot($destpath); ++ } ++ ++ echo "Importing snapshot data...\n"; ++ // Restore the database to snapshot state. ++ self::reset_database(); ++ ++ echo "Data imported. Resetting datarooot...\n"; ++ ++ // Clear current dataroot before restoring filedir from snapshot. ++ static::$datarootskiponreset = ['.', '..', 'phpunittestdir.txt', 'phpunit', '.htaccess', $snapshot, "$snapshot.zip"]; ++ self::reset_dataroot(); ++ self::reset_original_data(); ++ ++ $snapshotfiledir = $snapshotdir . DIRECTORY_SEPARATOR . 'filedir'; ++ $filedir = $dataroot . DIRECTORY_SEPARATOR . 'filedir'; ++ ++ // Copy snapshot filedir back to dataroot. ++ self::copy_filedir($snapshotfiledir, $filedir); ++ ++ initialise_cfg(); ++ ++ // Execute all adhoc tasks. ++ while ($task = \core\task\manager::get_next_adhoc_task(time())) { ++ $task->execute(); ++ \core\task\manager::adhoc_task_complete($task); ++ } ++ ++ set_config('upgraderunning', 0); ++ ++ self::store_database_state(); ++ self::$lastdbwrites = $DB->perf_get_writes(); ++ ++ remove_dir($snapshotdir); ++ unlink($zipfile); ++ ++ echo "Snapshot successfully restored.\n"; ++ } ++ ++ /** ++ * Upgrade the test site and upgrade/install any new plugins that were added. ++ * ++ * @return void may terminate execution with exit code ++ */ ++ public static function upgrade_site() { ++ ++ if (!self::is_test_site()) { ++ phpunit_bootstrap_error(PHPUNIT_EXITCODE_CONFIGERROR, 'Can not upgrade non-test site!!'); ++ } ++ ++ if (self::is_test_data_updated()) { ++ echo "No database changes detected, skipping upgrade.\n"; ++ return; ++ } ++ ++ initialise_cfg(); ++ upgrade_noncore(true); ++ set_config('upgraderunning', 0); ++ self::store_versions_hash(); ++ self::store_database_state(); ++ } ++ ++ /** ++ * Helper method that copies all contents from one directory to another, creating directories as needed. ++ * ++ * @param string $sourcedir the source directory to copy ++ * @param string $dirtocreate the destination directory to create and copy files to ++ * @return void ++ */ ++ private static function copy_filedir($sourcedir, $dirtocreate) { ++ ++ if (!file_exists($sourcedir)) { ++ echo "Source directory $sourcedir does not exist!\n"; ++ return; ++ } ++ ++ $iterator = new \RecursiveIteratorIterator( ++ new \RecursiveDirectoryIterator($sourcedir, \RecursiveDirectoryIterator::SKIP_DOTS), ++ \RecursiveIteratorIterator::SELF_FIRST ++ ); ++ ++ if (!is_dir($dirtocreate)) { ++ mkdir($dirtocreate, 0777, true); ++ } ++ ++ foreach ($iterator as $item) { ++ $destpath = $dirtocreate . DIRECTORY_SEPARATOR . $iterator->getSubPathName(); ++ if ($item->isDir()) { ++ if (!is_dir($destpath)) { ++ mkdir($destpath, 0777, true); ++ } ++ } else { ++ copy($item->getPathname(), $destpath); ++ } ++ } ++ } ++ ++ /** ++ * Create the database tables based on the stored snapshot structure. ++ * ++ * @param string $snapshotdir the target snapshot directory to create the tables from. ++ * @param bool $showprogress whether to show progress during table creation. ++ * @return void may terminate execution with exit code ++ */ ++ private static function create_tables_from_snapshot(string $snapshotdir, bool $showprogress = true) { ++ global $DB; ++ ++ $dbman = $DB->get_manager(); ++ ++ $structurefile = $snapshotdir . DIRECTORY_SEPARATOR . 'tablestructure.ser'; ++ if (!file_exists($structurefile)) { ++ phpunit_bootstrap_error(PHPUNIT_EXITCODE_CONFIGERROR, "Snapshot file $structurefile does not exist!!"); ++ } ++ $tablestructure = unserialize(file_get_contents($structurefile)); ++ if (!is_array($tablestructure)) { ++ phpunit_bootstrap_error(PHPUNIT_EXITCODE_CONFIGERROR, "Invalid tablestructure.ser in snapshot $snapshotdir"); ++ } ++ ++ $dotsonline = 0; ++ foreach ($tablestructure as $tablename => $columns) { ++ // If the table already exists, skip it. ++ if ($dbman->table_exists($tablename)) { ++ continue; ++ } ++ ++ $table = new \xmldb_table($tablename); ++ ++ // Build an xmldb_field for every column. ++ foreach ($columns as $colname => $col) { ++ $field = new \xmldb_field($colname); ++ $sequence = false; ++ // Map each column meta type to the appropriate xmldb type. ++ switch ($col->meta_type) { ++ case 'R': ++ $field->setType(XMLDB_TYPE_INTEGER); ++ $field->setLength(10); ++ $sequence = true; ++ break; ++ case 'I': ++ $field->setType(XMLDB_TYPE_INTEGER); ++ $field->setLength($col->max_length ?: 10); ++ break; ++ case 'N': ++ $floattypes = ['float', 'double', 'float4', 'float8', 'double precision']; ++ if (in_array(strtolower($col->type ?? ''), $floattypes)) { ++ $field->setType(XMLDB_TYPE_FLOAT); ++ } else { ++ $field->setType(XMLDB_TYPE_NUMBER); ++ $field->setLength($col->max_length ?: 10); ++ $field->setDecimals($col->scale ?: 0); ++ } ++ break; ++ case 'C': ++ $field->setType(XMLDB_TYPE_CHAR); ++ $field->setLength($col->max_length ?: 255); ++ break; ++ case 'X': ++ $field->setType(XMLDB_TYPE_TEXT); ++ break; ++ case 'B': ++ $field->setType(XMLDB_TYPE_BINARY); ++ break; ++ case 'L': ++ $field->setType(XMLDB_TYPE_INTEGER); ++ $field->setLength(1); ++ break; ++ default: ++ $field->setType(XMLDB_TYPE_INTEGER); ++ $field->setLength($col->max_length ?: 10); ++ } ++ $field->setNotNull($col->not_null); ++ if ($col->has_default && $col->default_value !== '') { ++ $field->setDefault($col->default_value); ++ } ++ if ($sequence) { ++ $field->setSequence(true); ++ } ++ $table->addField($field); ++ } ++ if (isset($columns['id'])) { ++ // Add a primary key for the 'id' column. ++ $key = new \xmldb_key('primary', XMLDB_KEY_PRIMARY, ['id']); ++ $table->addKey($key); ++ } ++ $dbman->create_table($table); ++ ++ if ($dotsonline == 60) { ++ if ($showprogress) { ++ echo "\n"; ++ } ++ $dotsonline = 0; ++ } ++ if ($showprogress) { ++ echo '.'; ++ } ++ $dotsonline += 1; ++ } ++ ++ echo "\nTables successfully created from snapshot.\n"; ++ } ++ + /** + * Builds root/phpunit.xml file using defaults from /phpunit.xml.dist + * +diff --git a/public/lib/classes/test/testing_util.php b/public/lib/classes/test/testing_util.php +index c3aa2a8f3cb..4ed9f7fc021 100644 +--- a/public/lib/classes/test/testing_util.php ++++ b/public/lib/classes/test/testing_util.php +@@ -140,6 +140,13 @@ abstract class testing_util { + return substr($classname, 0, strpos($classname, '_')); + } + ++ /** ++ * Reset the original data. ++ */ ++ protected static function reset_original_data() { ++ self::$originaldatafilesjsonadded = false; ++ } ++ + /** + * Get data generator + * +diff --git a/public/lib/phpunit/bootstrap.php b/public/lib/phpunit/bootstrap.php +index f49b8e614e7..48181e28342 100644 +--- a/public/lib/phpunit/bootstrap.php ++++ b/public/lib/phpunit/bootstrap.php +@@ -151,7 +151,7 @@ if (!is_writable($CFG->phpunit_dataroot)) { + if (!file_exists("$CFG->phpunit_dataroot/phpunittestdir.txt")) { + if ($dh = opendir($CFG->phpunit_dataroot)) { + while (($file = readdir($dh)) !== false) { +- if ($file === 'phpunit' || $file === '.' || $file === '..' || $file === '.DS_Store') { ++ if ($file === 'phpunit' || $file === '.' || $file === '..' || $file === '.DS_Store' || str_ends_with($file, '.zip')) { + continue; + } + phpunit_bootstrap_error( +-- +2.43.0 + diff --git a/.github/patches/README.md b/.github/patches/README.md new file mode 100644 index 00000000..dab3fbc5 --- /dev/null +++ b/.github/patches/README.md @@ -0,0 +1,43 @@ +# Shared Core Patches + +Patch files in this directory are applied to the Moodle source tree during CI setup, before +`moodle-plugin-ci install` runs. They are applied in alphabetical filename order using `git am`. + +## Naming convention + +``` +{min}-{max}-{description}.patch +``` + +| Part | Description | +|---|---| +| `min` | Lowest Moodle branch number this patch applies to (e.g. `500` for `MOODLE_500_STABLE`) | +| `max` | Highest Moodle branch number this patch applies to (e.g. `999` for `main`) | +| `description` | Freeform identifier (hyphens recommended) | + +The branch number is the `XXX` from `MOODLE_XXX_STABLE`. The `main` branch is treated as `999`. + +A patch is applied when `min <= current_branch_number <= max`. Patch files whose names do not +match the `NNN-NNN-` prefix are skipped with a warning. + +## Examples + +| Filename | Applies to | +|---|---| +| `500-999-phpunit-restore.patch` | Moodle 5.0 → main | +| `401-405-phpunit-restore.patch` | Moodle 4.1 → 4.5 | + +## Authoring patches + +Patches must be in `git format-patch` / `git am` format (i.e. include a commit header). To create +one: + +```bash +# Make your changes inside a Moodle checkout, then: +git add +git commit -m "Brief description of the patch" +git format-patch HEAD~1 --stdout > NNN-NNN-description.patch +``` + +The patch is applied with `git am --whitespace=nowarn`, so trailing-whitespace issues are ignored. +If `git am` fails the CI job fails immediately (fail-fast per matrix entry). diff --git a/.github/plugin/setup/action.yml b/.github/plugin/setup/action.yml index 2f52cb84..33837a6d 100644 --- a/.github/plugin/setup/action.yml +++ b/.github/plugin/setup/action.yml @@ -6,7 +6,8 @@ inputs: required: true node: description: 'Node.js version to use' - required: true + required: false + default: '' extra_php_extensions: description: 'List of additional php packages to install' extra_plugin_runners: @@ -34,6 +35,7 @@ runs: path: plugin submodules: true - name: Install node ${{ inputs.node }} + if: ${{ inputs.node != '' }} uses: actions/setup-node@v2 with: node-version: ${{ inputs.node }} @@ -46,72 +48,91 @@ runs: extensions: pgsql, mysqli, zip, gd, xmlrpc, soap, ${{ inputs.extra_php_extensions }} coverage: none - - name: Configure Required Composer Version - id: install-composer - if: ${{ inputs.moodle_branch == 'MOODLE_32_STABLE' || inputs.moodle_branch == 'MOODLE_33_STABLE' }} - run: | - echo "::set-output name=COMPOSER_VERSION::--1" + - name: Pull snapshot Docker image and extract artifacts shell: bash - - - name: Update Composer - if: ${{ inputs.moodle_branch == 'MOODLE_32_STABLE' || inputs.moodle_branch == 'MOODLE_33_STABLE' }} run: | - time composer self-update ${{ steps.install-composer1.outputs.COMPOSER_VERSION }} - shell: bash + BRANCH="${{ inputs.moodle_branch }}" + # Use provided database for image name; fall back to mariadb for jobs with no database. + DB="${{ inputs.database || 'mariadb' }}" + IMAGE="ghcr.io/catalyst/snapshot-${BRANCH}-${DB}" + IMAGE="${IMAGE//_/-}" + IMAGE="${IMAGE,,}:latest" + echo "SNAPSHOT_IMAGE=$IMAGE" >> "$GITHUB_ENV" + + START_TS=$(date +%s) + echo "::group::Pull snapshot Docker image" + docker pull "$IMAGE" + echo "::endgroup::" + END_TS=$(date +%s) + echo "Pull snapshot Docker image took $((END_TS - START_TS))s" + + START_TS=$(date +%s) + echo "::group::Extract artifacts" + CID=$(docker create "$IMAGE") + docker cp "$CID:/m-ci.tar.gz" ./m-ci.tar.gz + docker cp "$CID:/moodle.tar.gz" ./moodle.tar.gz + docker rm "$CID" + tar -C "$GITHUB_WORKSPACE" -xzf "$GITHUB_WORKSPACE/m-ci.tar.gz" + tar -C "$GITHUB_WORKSPACE" -xzf "$GITHUB_WORKSPACE/moodle.tar.gz" + mkdir -p "$GITHUB_WORKSPACE/moodledata" + echo "::endgroup::" + END_TS=$(date +%s) + echo "Extract artifacts took $((END_TS - START_TS))s" - - name: Initialise moodle-plugin-ci + - name: Add moodle-plugin-ci binaries to PATH run: | - # Initialise moodle-plugin-ci (install via composer) - echo "::group::Initialise moodle-plugin-ci" - - time composer create-project -n --no-dev --prefer-dist moodlehq/moodle-plugin-ci m-ci ^4 - # Add dirs to $PATH + START_TS=$(date +%s) echo $(cd m-ci/bin; pwd) >> $GITHUB_PATH echo $(cd m-ci/vendor/bin; pwd) >> $GITHUB_PATH + END_TS=$(date +%s) + echo "Add moodle-plugin-ci binaries to PATH took $((END_TS - START_TS))s" + shell: bash + + - name: Generate en_AU locale + run: | + START_TS=$(date +%s) # PHPUnit depends on en_AU.UTF-8 locale sudo locale-gen en_AU.UTF-8 - - echo "::endgroup::" + END_TS=$(date +%s) + echo "Generate en_AU locale took $((END_TS - START_TS))s" shell: bash - name: Install dependencies if: ${{ inputs.extra_plugin_runners }} run: | - # Install dependencies - echo "::group::Install dependencies" - + START_TS=$(date +%s) ${{ inputs.extra_plugin_runners }} - - echo "::endgroup::" + END_TS=$(date +%s) + echo "Install dependencies took $((END_TS - START_TS))s" shell: bash - - name: Clone Moodle - # Clone to a temporary directory + - name: Apply plugin branch patch run: | - echo "::group::Moodle clone output" - - time git clone https://github.com/moodle/moodle.git --branch $MOODLE_BRANCH $GITHUB_WORKSPACE/moodletemp + START_TS=$(date +%s) + TARGET_REPO="$GITHUB_WORKSPACE/moodle" + PATCH="$GITHUB_WORKSPACE/plugin/patch/${{ inputs.moodle_branch }}.diff" + + if [ -f "$PATCH" ]; then + # moodle/ was extracted from a tarball (no .git); init a repo just for patch application. + git config --global user.email "test@test.com" + git config --global user.name "Test" + git -C "$TARGET_REPO" init --quiet + git -C "$TARGET_REPO" apply --whitespace=nowarn "$PATCH" + else + echo "No plugin branch patch found for ${{ inputs.moodle_branch }}" + fi - echo "::endgroup::" + END_TS=$(date +%s) + echo "Apply plugin branch patch took $((END_TS - START_TS))s" shell: bash - env: - DB: ${{ inputs.database }} - MOODLE_BRANCH: ${{ inputs.moodle_branch }} - - name: Install Core Patches - if: ${{ always() }} - # Install core patches to moodle in temporary directory - run: | - git config --global user.email "test@test.com" - git config --global user.name "Test" - ((test -f plugin/patch/${{ inputs.moodle_branch }}.diff && cd $GITHUB_WORKSPACE/moodletemp && git am --whitespace=nowarn < ../plugin/patch/${{ inputs.moodle_branch }}.diff) || echo No patch found;) + - name: Install plugin into Moodle shell: bash - - - name: Install plugin into Moodle (no database) - if: ${{ inputs.database == '' }} - # For static analysis jobs that don't need a database, copy the plugin into the Moodle - # directory so that tools like mustache and grunt can find it within Moodle's directory tree. run: | + if [ ! -f "plugin/version.php" ]; then + echo "No plugin/version.php found, skipping plugin install." + exit 0 + fi COMPONENT=$(php -r " define('MOODLE_INTERNAL', true); define('MATURITY_ALPHA', 50); @@ -125,59 +146,16 @@ runs: TYPE=$(echo "$COMPONENT" | cut -d_ -f1) NAME=$(echo "$COMPONENT" | cut -d_ -f2-) TYPE_DIR=$(php -r " - \$components = json_decode(file_get_contents('$GITHUB_WORKSPACE/moodletemp/lib/components.json'), true); + \$components = json_decode(file_get_contents('$GITHUB_WORKSPACE/moodle/lib/components.json'), true); \$type = '$TYPE'; if (!isset(\$components['plugintypes'][\$type])) { fwrite(STDERR, 'Unknown plugin type: ' . \$type . PHP_EOL); exit(1); } echo \$components['plugintypes'][\$type]; ") - PLUGIN_PATH="$GITHUB_WORKSPACE/moodletemp/$TYPE_DIR/$NAME" + PLUGIN_PATH="$GITHUB_WORKSPACE/moodle/$TYPE_DIR/$NAME" mkdir -p "$PLUGIN_PATH" cp -r plugin/. "$PLUGIN_PATH/" rm -rf "$PLUGIN_PATH/.git" - # Create a minimal config.php so tools that bootstrap Moodle (e.g. validate) can load - # core_component without connecting to a database (ABORT_AFTER_CONFIG is set by moodle-plugin-ci). - mkdir -p /tmp/moodledata - php -r " - file_put_contents('$GITHUB_WORKSPACE/moodletemp/config.php', - implode(PHP_EOL, [ - 'dbtype = \'pgsql\';', - '\$CFG->dblibrary = \'native\';', - '\$CFG->dbhost = \'fakehost\';', - '\$CFG->dbname = \'fakedb\';', - '\$CFG->dbuser = \'fakeuser\';', - '\$CFG->dbpass = \'\';', - '\$CFG->prefix = \'mdl_\';', - '\$CFG->dboptions = [];', - '\$CFG->wwwroot = \'http://localhost\';', - '\$CFG->dataroot = \'/tmp/moodledata\';', - '\$CFG->admin = \'admin\';', - '\$CFG->directorypermissions = 0777;', - 'require_once(__DIR__ . \'/lib/setup.php\');', - ]) - ); - " - echo "MOODLE_DIR=$GITHUB_WORKSPACE/moodletemp" >> "$GITHUB_ENV" + echo "MOODLE_DIR=$GITHUB_WORKSPACE/moodle" >> "$GITHUB_ENV" echo "PLUGIN_DIR=$PLUGIN_PATH" >> "$GITHUB_ENV" - shell: bash - - - name: Install Moodle and Plugin - if: ${{ inputs.database != '' }} - # Install moodle, but use our temporary directory to include any potential core patches that have been applied. - run: | - # Install moodle commands - echo "::group::Moodle install output" - - # This is a workaround for https://github.com/moodlehq/moodle-plugin-ci/issues/306 - we disable the git repository validation to allow folder cloning. - sed -i '/public function gitUrl($url)/{n; s/{/& return $url;/}' $GITHUB_WORKSPACE/m-ci/src/Validate.php - time moodle-plugin-ci install -vvv --plugin ./plugin --db-host=127.0.0.1 --repo $GITHUB_WORKSPACE/moodletemp - - echo "::endgroup::" - shell: bash - env: - DB: ${{ inputs.database }} - MOODLE_BRANCH: ${{ inputs.moodle_branch }} diff --git a/.github/templates/moodle-config-template.php b/.github/templates/moodle-config-template.php new file mode 100644 index 00000000..08761f55 --- /dev/null +++ b/.github/templates/moodle-config-template.php @@ -0,0 +1,36 @@ +dbtype = '{{DBTYPE}}'; +$CFG->dblibrary = 'native'; +$CFG->dbhost = '127.0.0.1'; +$CFG->dbname = 'moodle'; +$CFG->dbuser = '{{DBTYPE}}' === 'pgsql' ? 'postgres' : 'root'; +$CFG->dbpass = ''; +$CFG->prefix = 'mdl_'; +$CFG->dboptions = [ + 'dbpersist' => 0, + 'dbport' => '', + 'dbsocket' => '', + 'dbcollation' => 'utf8mb4_unicode_ci', +]; + +// config.php lives at moodle/config.php; dataroot is a sibling of moodle/. +$CFG->wwwroot = 'http://localhost'; +$CFG->dataroot = dirname(__DIR__) . '/moodledata'; +$CFG->admin = 'admin'; +$CFG->directorypermissions = 02777; + +$CFG->phpunit_prefix = 'phpu_'; +$CFG->phpunit_dataroot = dirname(__DIR__) . '/moodledata/phpunit'; + +// Support both classic layout (lib/setup.php) and public/ layout (public/lib/setup.php). +if (file_exists(__DIR__ . '/public/lib/setup.php')) { + require_once(__DIR__ . '/public/lib/setup.php'); +} else { + require_once(__DIR__ . '/lib/setup.php'); +} diff --git a/.github/workflows/build-dockerfiles.yml b/.github/workflows/build-dockerfiles.yml deleted file mode 100644 index e5cbff88..00000000 --- a/.github/workflows/build-dockerfiles.yml +++ /dev/null @@ -1,88 +0,0 @@ -name: Build Dockerfiles - -on: - workflow_dispatch: - push: - branches: - - main - paths: - - '.github/actions/matrix/matrix_includes.yml' - - 'docker/**' - - '.github/workflows/build-dockerfiles.yml' - -jobs: - build-matrix: - runs-on: ubuntu-latest - outputs: - matrix: ${{ steps.set-matrix.outputs.matrix }} - steps: - - name: Checkout repository - uses: actions/checkout@v4 - - - name: Set up Python (for YAML parsing) - uses: actions/setup-python@v5 - with: - python-version: '3.x' - - - name: Install PyYAML - run: pip install pyyaml - - - name: Extract unique moodle-branch/php matrix - id: set-matrix - run: | - import yaml - import json - with open('.github/actions/matrix/matrix_includes.yml') as f: - data = yaml.safe_load(f) - pairs = set((row['moodle-branch'], row['php']) for row in data['include']) - - def container_name(branch, php): - name = f"catalyst-moodle-workflows-{branch}-{php}" - name = name.replace('_', '-').lower() - return f"ghcr.io/catalyst/{name}:latest" - - def find_node(branch, php): - for row in data['include']: - if row['moodle-branch'] == branch and row['php'] == php: - return row.get('node', 20) - return 20 - - matrix = [ - { - 'moodle-branch': b, - 'php': p, - 'node': find_node(b, p), - 'container': container_name(b, p) - } - for b, p in sorted(pairs) - ] - print('::set-output name=matrix::' + json.dumps(matrix)) - shell: python - - build: - name: ${{ matrix.container }} - needs: build-matrix - runs-on: ubuntu-latest - strategy: - matrix: - include: ${{fromJson(needs.build-matrix.outputs.matrix)}} - steps: - - name: Checkout repository - uses: actions/checkout@v4 - - name: Log in to GitHub Container Registry - uses: docker/login-action@v3 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - name: Build Docker image - run: | - docker build \ - --build-arg MOODLE_BRANCH=${{ matrix.moodle-branch }} \ - --build-arg PHP_VERSION=${{ matrix.php }} \ - --build-arg NODE_VERSION=${{ matrix.node }} \ - -t ${{ matrix.container }} \ - -f docker/Dockerfile docker/ - - - name: Push Docker image to GHCR - run: docker push ${{ matrix.container }} diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 936b0453..8722df9d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -516,7 +516,7 @@ jobs: runs-on: 'ubuntu-latest' services: postgres: - image: "postgres:${{ matrix.pgsql-ver }}" + image: ${{ matrix.pgsql-ver && format('postgres:{0}', matrix.pgsql-ver) || '' }} env: POSTGRES_USER: 'postgres' POSTGRES_HOST_AUTH_METHOD: 'trust' @@ -528,7 +528,7 @@ jobs: ports: - 5432:5432 mariadb: - image: "mariadb:${{ matrix.mariadb-ver }}" + image: ${{ matrix.mariadb-ver && format('mariadb:{0}', matrix.mariadb-ver) || '' }} env: MYSQL_USER: 'root' MYSQL_ALLOW_EMPTY_PASSWORD: "true" @@ -549,21 +549,71 @@ jobs: repository: catalyst/catalyst-moodle-workflows ref: ${{ inputs.internal_workflow_branch }} token: ${{ github.token }} + - name: Tune PostgreSQL for CI speed + if: matrix.database == 'pgsql' + run: | + psql -h 127.0.0.1 -U postgres -c "ALTER SYSTEM SET fsync = off" + psql -h 127.0.0.1 -U postgres -c "ALTER SYSTEM SET synchronous_commit = off" + psql -h 127.0.0.1 -U postgres -c "ALTER SYSTEM SET full_page_writes = off" + psql -h 127.0.0.1 -U postgres -c "ALTER SYSTEM SET checkpoint_completion_target = 0.9" + psql -h 127.0.0.1 -U postgres -c "ALTER SYSTEM SET max_wal_size = '1GB'" + psql -h 127.0.0.1 -U postgres -c "SELECT pg_reload_conf()" - name: Run plugin setup uses: ./ci/.github/plugin/setup with: php: ${{ matrix.php }} - node: ${{ matrix.node }} extra_php_extensions: ${{ inputs.extra_php_extensions }} extra_plugin_runners: ${{ inputs.extra_plugin_runners }} moodle_branch: ${{ matrix.moodle-branch }} database: ${{ matrix.database }} + - name: Create database and restore PHPUnit snapshot + run: | + START_TS=$(date +%s) + + # Extract phpunit snapshot zip from the image — fail if missing. + CID=$(docker create "$SNAPSHOT_IMAGE") + docker cp "$CID:/phpunit-snapshot.zip" ./phpunit-snapshot.zip + docker rm "$CID" + + # Create the Moodle database. + if [ "${{ matrix.database }}" = "pgsql" ]; then + psql -h 127.0.0.1 -U postgres -d postgres -c "DROP DATABASE IF EXISTS moodle;" + psql -h 127.0.0.1 -U postgres -d postgres -c "CREATE DATABASE moodle;" + else + mysql -h 127.0.0.1 -u root -e "DROP DATABASE IF EXISTS moodle; CREATE DATABASE moodle DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;" + fi + + # phpunit_dataroot is always $GITHUB_WORKSPACE/moodledata/phpunit per the config template. + PHPUNIT_DATAROOT="$GITHUB_WORKSPACE/moodledata/phpunit" + mkdir -p "$PHPUNIT_DATAROOT" + cp "$GITHUB_WORKSPACE/phpunit-snapshot.zip" "$PHPUNIT_DATAROOT/ci-phpunit.zip" + + # Detect phpunit util location (differs between classic and public/ layouts). + if [ -f "$GITHUB_WORKSPACE/moodle/public/admin/tool/phpunit/cli/util.php" ]; then + PHPUNIT_UTIL="$GITHUB_WORKSPACE/moodle/public/admin/tool/phpunit/cli/util.php" + elif [ -f "$GITHUB_WORKSPACE/moodle/admin/tool/phpunit/cli/util.php" ]; then + PHPUNIT_UTIL="$GITHUB_WORKSPACE/moodle/admin/tool/phpunit/cli/util.php" + else + echo "Could not locate phpunit util.php" + exit 1 + fi + + php "$PHPUNIT_UTIL" --restore=ci-phpunit + php "$PHPUNIT_UTIL" --upgrade + php "$PHPUNIT_UTIL" --buildconfig + + END_TS=$(date +%s) + echo "Create database and restore PHPUnit snapshot took $((END_TS - START_TS))s" + shell: bash - name: Run phpunit + working-directory: moodle run: | - moodle-plugin-ci phpunit - cd moodle - vendor/bin/phpunit --fail-on-risky --disallow-test-output --filter tool_dataprivacy_metadata_registry_testcase - vendor/bin/phpunit --fail-on-risky --disallow-test-output --testsuite core_privacy_testsuite --filter provider_testcase + echo "::group::Plugin unit tests" + vendor/bin/phpunit --fail-on-risky --disallow-test-output --testsuite ${{ needs.prepare_matrix.outputs.component }}_testsuite + echo "::endgroup::" + echo "::group::Privacy unit tests" + vendor/bin/phpunit --fail-on-risky --disallow-test-output --testsuite core_privacy_testsuite --filter test_all_providers_compliant + echo "::endgroup::" shell: bash behat: name: ${{ matrix.moodle-branch-short }} - behat - php${{ matrix.php }} - ${{ matrix.database }} @@ -580,7 +630,7 @@ jobs: runs-on: 'ubuntu-latest' services: postgres: - image: "postgres:${{ matrix.pgsql-ver }}" + image: ${{ matrix.pgsql-ver && format('postgres:{0}', matrix.pgsql-ver) || '' }} env: POSTGRES_USER: 'postgres' POSTGRES_HOST_AUTH_METHOD: 'trust' @@ -592,7 +642,7 @@ jobs: ports: - 5432:5432 mariadb: - image: "mariadb:${{ matrix.mariadb-ver }}" + image: ${{ matrix.mariadb-ver && format('mariadb:{0}', matrix.mariadb-ver) || '' }} env: MYSQL_USER: 'root' MYSQL_ALLOW_EMPTY_PASSWORD: "true" diff --git a/.github/workflows/update-snapshots.yml b/.github/workflows/update-snapshots.yml index 2bb1ca7e..497bc6d7 100644 --- a/.github/workflows/update-snapshots.yml +++ b/.github/workflows/update-snapshots.yml @@ -4,6 +4,11 @@ on: workflow_dispatch: schedule: - cron: '0 0 * * 0' # Every Sunday at midnight UTC + push: + branches: + - main + paths: + - .github/actions/matrix/matrix_includes.yml permissions: packages: write @@ -111,69 +116,158 @@ jobs: with: path: ci - - name: Run setup (PHP, Node, moodle-plugin-ci, Moodle clone, shared patches) - # database is intentionally empty — we handle the Moodle install manually below - # so we can point moodle-plugin-ci at our minimal snapshot-plugin instead of - # this CI repo (which the composite action would check out as ./plugin/). - uses: ./ci/.github/plugin/setup + - name: Install node ${{ matrix.node }} + uses: actions/setup-node@v4 with: - php: ${{ matrix.php }} - node: ${{ matrix.node }} - moodle_branch: ${{ matrix.moodle-branch }} - database: '' + node-version: ${{ matrix.node }} - - name: Install Moodle with snapshot placeholder plugin + - name: Setup PHP ${{ matrix.php }} + uses: shivammathur/setup-php@9e72090525849c5e82e596468b86eb55e9cc5401 # v2.32.0 + with: + php-version: ${{ matrix.php }} + ini-values: max_input_vars=5000 + extensions: pgsql, mysqli, zip, gd, xmlrpc, soap + coverage: none + + - name: Install moodle-plugin-ci + shell: bash + run: | + composer create-project -n --no-dev --prefer-dist moodlehq/moodle-plugin-ci m-ci ^4 + echo "$(cd m-ci/bin; pwd)" >> "$GITHUB_PATH" + echo "$(cd m-ci/vendor/bin; pwd)" >> "$GITHUB_PATH" + + - name: Clone Moodle for snapshot shell: bash + run: | + git clone https://github.com/moodle/moodle.git --branch "$MOODLE_BRANCH" "$GITHUB_WORKSPACE/moodle" env: - DB: ${{ matrix.database }} MOODLE_BRANCH: ${{ matrix.moodle-branch }} + + - name: Apply shared core patches + shell: bash run: | - # The composite setup action checked out this CI repo into ./plugin/. - # Replace it with a minimal inline Moodle plugin so that - # moodle-plugin-ci has a valid plugin to install against. - rm -rf ./plugin - mkdir -p ./plugin/lang/en - printf 'component = "local_snapshottest";\n$plugin->version = 2026010100;\n$plugin->requires = 2021051700;\n' \ - > ./plugin/version.php - printf ' ./plugin/lang/en/local_snapshottest.php - - # Workaround for https://github.com/moodlehq/moodle-plugin-ci/issues/306 - # (disables git URL validation to allow folder-based repo cloning). - sed -i '/public function gitUrl($url)/{n; s/{/& return $url;/}' \ - "$GITHUB_WORKSPACE/m-ci/src/Validate.php" - - moodle-plugin-ci install -vvv \ - --plugin ./plugin \ - --db-host=127.0.0.1 \ - --repo "$GITHUB_WORKSPACE/moodletemp" - - - name: Initialise PHPUnit and export database + git config --global user.email "test@test.com" + git config --global user.name "Test" + + MOODLE_BRANCH="${{ matrix.moodle-branch }}" + if [[ "$MOODLE_BRANCH" == "main" ]]; then + BRANCH_NUM=999 + elif [[ "$MOODLE_BRANCH" =~ MOODLE_([0-9]+)_STABLE ]]; then + BRANCH_NUM=${BASH_REMATCH[1]} + else + echo "Could not parse branch number from '$MOODLE_BRANCH', skipping shared patches." + exit 0 + fi + + PATCHES_DIR="${GITHUB_WORKSPACE}/ci/.github/patches" + if [ ! -d "$PATCHES_DIR" ]; then + echo "No shared patches directory found at $PATCHES_DIR, skipping." + exit 0 + fi + + shopt -s nullglob + PATCH_FILES=( "$PATCHES_DIR"/*.patch ) + if [ ${#PATCH_FILES[@]} -eq 0 ]; then + echo "No patch files found in $PATCHES_DIR" + exit 0 + fi + + for PATCHFILE in "${PATCH_FILES[@]}"; do + BASENAME=$(basename "$PATCHFILE") + if [[ "$BASENAME" =~ ^([0-9]+)-([0-9]+)- ]]; then + MIN_VER=${BASH_REMATCH[1]} + MAX_VER=${BASH_REMATCH[2]} + if (( BRANCH_NUM < MIN_VER || BRANCH_NUM > MAX_VER )); then + echo "Skipping $BASENAME (branch $BRANCH_NUM not in range $MIN_VER-$MAX_VER)" + continue + fi + else + echo "WARNING: patch $BASENAME has no version prefix (expected NNN-NNN-name.patch), skipping." + continue + fi + + echo "Applying shared patch $BASENAME to branch $MOODLE_BRANCH..." + cd "$GITHUB_WORKSPACE/moodle" + git am --whitespace=nowarn < "$PATCHFILE" + done + + - name: Install Moodle composer dependencies + shell: bash + run: | + composer --working-dir="moodle" install --no-interaction --prefer-dist + + - name: Install PHPUnit DB for snapshot (no plugin install) shell: bash env: DB: ${{ matrix.database }} run: | - # Moodle 5.0+ uses a public/ webroot subdirectory; 4.x does not. - if [ -d moodle/public ]; then - UTIL="moodle/public/admin/tool/phpunit/cli/util.php" + if [ "$DB" = "pgsql" ]; then + DBTYPE="pgsql" + psql -h 127.0.0.1 -U postgres -d postgres -c "DROP DATABASE IF EXISTS moodle;" + psql -h 127.0.0.1 -U postgres -d postgres -c "CREATE DATABASE moodle;" + else + DBTYPE="mariadb" + mysql -h 127.0.0.1 -u root -e "DROP DATABASE IF EXISTS moodle; CREATE DATABASE moodle DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;" + fi + + # Render config.php from template — only DBTYPE varies. + sed "s|{{DBTYPE}}|$DBTYPE|g" \ + "$GITHUB_WORKSPACE/ci/.github/templates/moodle-config-template.php" > moodle/config.php + + # Detect phpunit CLI location (differs between classic and public/ layouts). + if [ -f "moodle/public/admin/tool/phpunit/cli/init.php" ]; then + PHPUNIT_CLI="moodle/public/admin/tool/phpunit/cli" + elif [ -f "moodle/admin/tool/phpunit/cli/init.php" ]; then + PHPUNIT_CLI="moodle/admin/tool/phpunit/cli" else - UTIL="moodle/admin/tool/phpunit/cli/util.php" + echo "Could not locate Moodle phpunit init script in moodle/ or moodle/public/." + exit 1 fi - echo "Initialising PHPUnit test site..." - php "$UTIL" --install + sudo locale-gen en_AU.UTF-8 + mkdir -p "$GITHUB_WORKSPACE/moodledata/phpunit" + php "$PHPUNIT_CLI/init.php" - # Read the PHPUnit database name from Moodle's config. - PHPUNIT_DBNAME=$(php -r "define('CLI_SCRIPT', true); require 'moodle/config.php'; echo \$CFG->phpunit_dbname;") - echo "Exporting PHPUnit database '$PHPUNIT_DBNAME'..." + - name: Archive phpunit-ready Moodle tree + shell: bash + run: | + rm -rf moodle/.git + tar -C "$GITHUB_WORKSPACE" -czf "$GITHUB_WORKSPACE/moodle.tar.gz" moodle + echo "MOODLE_ARCHIVE=$GITHUB_WORKSPACE/moodle.tar.gz" >> "$GITHUB_ENV" - if [ "$DB" = "pgsql" ]; then - pg_dump -h 127.0.0.1 -U postgres "$PHPUNIT_DBNAME" > snapshot.sql + - name: Export phpunit snapshot + shell: bash + run: | + # Detect util.php location (differs between classic and public/ layouts). + if [ -f "$GITHUB_WORKSPACE/moodle/public/admin/tool/phpunit/cli/util.php" ]; then + PHPUNIT_UTIL="$GITHUB_WORKSPACE/moodle/public/admin/tool/phpunit/cli/util.php" + elif [ -f "$GITHUB_WORKSPACE/moodle/admin/tool/phpunit/cli/util.php" ]; then + PHPUNIT_UTIL="$GITHUB_WORKSPACE/moodle/admin/tool/phpunit/cli/util.php" else - mysqldump -h 127.0.0.1 -u root "$PHPUNIT_DBNAME" > snapshot.sql + echo "Could not locate phpunit util.php" + exit 1 fi - echo "SNAPSHOT_SQL=$GITHUB_WORKSPACE/snapshot.sql" >> "$GITHUB_ENV" + cd moodle + php "$PHPUNIT_UTIL" --snapshot=ci-phpunit + + # phpunit_dataroot is always $GITHUB_WORKSPACE/moodledata/phpunit per the config template. + PHPUNIT_DATAROOT="$GITHUB_WORKSPACE/moodledata/phpunit" + PHPUNIT_SNAPSHOT=$(find "$PHPUNIT_DATAROOT" -maxdepth 1 -type f -name '*-ci-phpunit-*.zip' | head -n1) + + if [ -z "$PHPUNIT_SNAPSHOT" ]; then + echo "Could not find phpunit snapshot archive in $PHPUNIT_DATAROOT" + exit 1 + fi + + cp "$PHPUNIT_SNAPSHOT" "$GITHUB_WORKSPACE/phpunit-snapshot.zip" + echo "PHPUNIT_SNAPSHOT=$GITHUB_WORKSPACE/phpunit-snapshot.zip" >> "$GITHUB_ENV" + + - name: Archive moodle-plugin-ci + shell: bash + run: | + tar -C "$GITHUB_WORKSPACE" -czf "$GITHUB_WORKSPACE/m-ci.tar.gz" m-ci + echo "MCI_ARCHIVE=$GITHUB_WORKSPACE/m-ci.tar.gz" >> "$GITHUB_ENV" - name: Log in to GitHub Container Registry uses: docker/login-action@v3 @@ -188,8 +282,10 @@ jobs: IMAGE="${{ matrix.snapshot-image }}" BUILD_DIR=$(mktemp -d) - cp "$SNAPSHOT_SQL" "$BUILD_DIR/snapshot.sql" - printf 'FROM alpine:3\nCOPY snapshot.sql /snapshot.sql\n' > "$BUILD_DIR/Dockerfile" + cp "$PHPUNIT_SNAPSHOT" "$BUILD_DIR/phpunit-snapshot.zip" + cp "$MOODLE_ARCHIVE" "$BUILD_DIR/moodle.tar.gz" + cp "$MCI_ARCHIVE" "$BUILD_DIR/m-ci.tar.gz" + printf 'FROM alpine:3\nCOPY phpunit-snapshot.zip /phpunit-snapshot.zip\nCOPY moodle.tar.gz /moodle.tar.gz\nCOPY m-ci.tar.gz /m-ci.tar.gz\n' > "$BUILD_DIR/Dockerfile" docker build -t "$IMAGE" "$BUILD_DIR" docker push "$IMAGE"