From c359ef9c4d017cd52f4c85f72f7ce696b6e3cb87 Mon Sep 17 00:00:00 2001 From: ReenigneArcher <42013603+ReenigneArcher@users.noreply.github.com> Date: Sun, 8 Nov 2026 17:42:08 -0500 Subject: [PATCH 1/4] ci: support multiple organizations in central maintenance --- .github/repositories-to-ignore.json | 8 +- .github/scripts/gitlab-mirror.mjs | 208 +++++++ .github/scripts/mirror-gitlab.sh | 490 +++++++++++++++ .github/scripts/organization-repositories.mjs | 85 +++ .github/workflows/README.md | 31 +- .github/workflows/__issues-stale.yml | 34 +- .../workflows/__mirror-github-to-gitlab.yml | 557 +----------------- .github/workflows/__pinact.yml | 55 +- .github/workflows/__update-build-deps.yml | 25 +- .github/workflows/__update-cpm-cmake.yml | 34 +- .github/workflows/__update_license_years.yml | 32 +- tests/gitlab-mirror.test.mjs | 167 ++++++ tests/organization-repositories.test.mjs | 108 ++++ tests/workflow-profile-selection.test.mjs | 47 +- 14 files changed, 1225 insertions(+), 656 deletions(-) create mode 100644 .github/scripts/gitlab-mirror.mjs create mode 100644 .github/scripts/mirror-gitlab.sh create mode 100644 .github/scripts/organization-repositories.mjs create mode 100644 tests/gitlab-mirror.test.mjs create mode 100644 tests/organization-repositories.test.mjs diff --git a/.github/repositories-to-ignore.json b/.github/repositories-to-ignore.json index 7488f5d2..e16494e1 100644 --- a/.github/repositories-to-ignore.json +++ b/.github/repositories-to-ignore.json @@ -1,6 +1,6 @@ [ - "homebrew-core", - "winget-pkgs", - "Virtual-Gamepad-Emulation-Client", - "Virtual-Gamepad-Emulation-dotnet" + "LizardByte/homebrew-core", + "LizardByte/winget-pkgs", + "LizardByte/Virtual-Gamepad-Emulation-Client", + "LizardByte/Virtual-Gamepad-Emulation-dotnet" ] diff --git a/.github/scripts/gitlab-mirror.mjs b/.github/scripts/gitlab-mirror.mjs new file mode 100644 index 00000000..880038a4 --- /dev/null +++ b/.github/scripts/gitlab-mirror.mjs @@ -0,0 +1,208 @@ +import {readFileSync, writeFileSync} from 'node:fs'; +import {pathToFileURL} from 'node:url'; + +import {forEachSequential} from './async-iteration.mjs'; + +/** Apply qualified mirror exclusions and omit temporary security-advisory forks. */ +export function mirrorRepositories(repos, ignored, rootGroup, sourceOrganization) { + if (!Array.isArray(ignored) || !ignored.every(value => + typeof value === 'string' && /^[\w-]+\/[\w.-]+$/.test(value))) { + throw new Error('Mirror exclusions must be qualified owner/repository names'); + } + const exclusions = new Set(ignored.map(value => value.toLowerCase())); + const segment = '[23456789cfghjmpqrvwx]{4}'; + const advisoryFork = new RegExp(`-ghsa-${segment}-${segment}-${segment}$`, 'i'); + return repos.filter(repo => !exclusions.has(repo.full_name.toLowerCase()) && !advisoryFork.test(repo.name)) + .map(repo => mirrorTarget(repo, rootGroup, sourceOrganization)); +} + +/** Keep existing LizardByte paths; isolate other GitHub owners in matching GitLab subgroups. */ +export function mirrorTarget(repo, rootGroup, sourceOrganization) { + if (repo.clone_url !== `https://github.com/${repo.full_name}.git` || + !Number.isSafeInteger(repo.size) || repo.size < 0) throw new Error('Invalid mirror source metadata'); + const prefixNames = {'.': 'dot-', '-': 'dash-', '_': 'underscore-'}; + const lowerName = repo.name.toLowerCase(); + let path = lowerName.replace(/^[._-]+/, prefix => [...prefix].map(char => prefixNames[char]).join('')) + .replace(/[^a-z0-9_.-]+/g, '-').replace(/[._-]{2,}/g, '-').replace(/[._-]+$/g, ''); + if (path.endsWith('.git') || path.endsWith('.atom')) path += '-repo'; + if (!path) path = 'repository'; + const transformed = path !== lowerName; + const targetGroup = repo.owner.login.toLowerCase() === sourceOrganization.toLowerCase() + ? rootGroup : `${rootGroup}/${repo.owner.login.toLowerCase()}`; + return { + id: repo.id, name: repo.full_name, cloneUrl: repo.clone_url, sizeKiB: repo.size, + targetName: transformed ? path : repo.name, + targetPath: transformed ? `${path}-${repo.id}` : repo.name, + targetGroup, targetVisibility: repo.visibility === 'public' ? 'public' : 'private', + description: `Mirror of ${repo.clone_url}\nGitHub repository ID: ${repo.id}`, + }; +} + +/** Reconcile namespaces and transfer verified legacy mirrors without deleting or recreating projects. */ +export class GitLabMirrors { + constructor({gitlabToken, githubToken, rootGroup = 'lizardbyte', fetchImpl = fetch, + sleep = milliseconds => new Promise(resolve => setTimeout(resolve, milliseconds))}) { + this.gitlabToken = gitlabToken; + this.githubToken = githubToken; + this.rootGroup = rootGroup; + this.fetch = fetchImpl; + this.sleep = sleep; + this.groups = new Map(); + } + + /** Make an authenticated request without following a renamed project's redirect or logging credentials. */ + async request(method, path, body) { + const response = await this.fetch(`https://gitlab.com/api/v4${path}`, { + method, redirect: 'manual', signal: AbortSignal.timeout(60000), + headers: {'PRIVATE-TOKEN': this.gitlabToken, 'Content-Type': 'application/json'}, + ...(body ? {body: JSON.stringify(body)} : {}), + }); + const data = await response.json(); + return {status: response.status, data}; + } + + /** Read the root or create one direct subgroup; never attempt unsupported top-level group creation. */ + async group(path, displayName) { + if (this.groups.has(path)) return this.groups.get(path); + let result = await this.request('GET', `/groups/${encodeURIComponent(path)}`); + if (result.status === 404 && path !== this.rootGroup) { + result = await this.createSubgroup(path, displayName); + } else if (result.status !== 200) { + throw new Error(`Cannot read GitLab group (HTTP ${result.status})`); + } + if (!Number.isSafeInteger(result.data.id) || result.data.full_path !== path || + !['public', 'internal', 'private'].includes(result.data.visibility)) { + throw new Error('GitLab group verification failed'); + } + if (path !== this.rootGroup) { + const parent = await this.group(this.rootGroup); + if (result.data.parent_id !== parent.id) throw new Error('GitLab subgroup parent verification failed'); + } + this.groups.set(path, result.data); + return result.data; + } + + /** Create a direct child and tolerate another owner creating the same subgroup concurrently. */ + async createSubgroup(path, displayName) { + const parent = await this.group(this.rootGroup); + const name = path.slice(this.rootGroup.length + 1); + if (!/^[a-z\d][a-z\d-]*$/i.test(name)) throw new Error('Invalid GitLab subgroup path'); + let result = await this.request('POST', '/groups', { + parent_id: parent.id, name: displayName ?? name, path: name, + visibility: parent.visibility, auto_devops_enabled: false, + }); + if ([400, 409].includes(result.status)) result = await this.request('GET', `/groups/${encodeURIComponent(path)}`); + if (![200, 201].includes(result.status)) throw new Error(`Cannot create GitLab subgroup (HTTP ${result.status})`); + return result; + } + + /** Verify provenance by immutable GitHub ID, resolving legacy clone URLs through GitHub redirects. */ + async verifyIdentity(project, repo, allowDifferent = false) { + if (!Number.isSafeInteger(project.id) || project.id <= 0) throw new Error('Invalid GitLab project identity'); + const description = project.description ?? ''; + const marker = /^GitHub repository ID: (\d+)$/m.exec(description); + if (marker) { + if (Number(marker[1]) === repo.id) return true; + if (allowDifferent) return false; + throw new Error('GitLab mirror belongs to a different GitHub repository'); + } + const source = /^Mirror of https:\/\/github\.com\/([\w-]+)\/([\w.-]+)\.git(?:\n|$)/.exec(description); + if (!source) throw new Error('Existing GitLab project has no verifiable mirror identity'); + const response = await this.fetch(`https://api.github.com/repos/${source[1]}/${source[2]}`, { + headers: {Authorization: `Bearer ${this.githubToken}`, Accept: 'application/vnd.github+json'}, + signal: AbortSignal.timeout(30000), + }); + if (!response.ok) throw new Error('Cannot verify the existing mirror through GitHub'); + const id = (await response.json()).id; + if (id === repo.id) return true; + if (Number.isSafeInteger(id) && allowDifferent) return false; + throw new Error('Existing GitLab mirror identity does not match the source repository'); + } + + /** Secure non-public sources before transferring or allowing the Git push stage to proceed. */ + async secure(project, repo) { + if (repo.targetVisibility !== 'private') return; + const result = await this.request('PUT', `/projects/${project.id}`, {visibility: 'private'}); + if (result.status !== 200 || result.data.id !== project.id || result.data.visibility !== 'private') { + throw new Error('GitLab privacy verification failed'); + } + } + + /** Resolve an existing mirror or a new-project namespace, preserving legacy GitLab project IDs. */ + async prepare(repo) { + const group = await this.group(repo.targetGroup, repo.name.split('/')[0]); + if (repo.targetVisibility === 'public' && group.visibility !== 'public') { + throw new Error('Public mirrors require a public GitLab group'); + } + const desired = `${repo.targetGroup}/${repo.targetPath}`; + const result = await this.request('GET', `/projects/${encodeURIComponent(desired)}`); + let projectId = null; + if (result.status === 200) { + if (result.data.path_with_namespace !== desired || result.data.namespace?.id !== group.id) { + throw new Error('GitLab project namespace verification failed'); + } + await this.verifyIdentity(result.data, repo); + if (result.data.archived) throw new Error('Existing GitLab mirror is archived and read-only'); + await this.secure(result.data, repo); + projectId = result.data.id; + } else if (![301, 404].includes(result.status)) { + throw new Error(`Cannot read GitLab mirror (HTTP ${result.status})`); + } else if (repo.targetGroup !== this.rootGroup) { + projectId = await this.transferLegacy(repo, group); + } + return {...repo, namespaceId: group.id, projectId}; + } + + /** Find an original root mirror, verify it, then move that project rather than replacing its history. */ + async transferLegacy(repo, group) { + const legacyPath = `${this.rootGroup}/${repo.targetPath}`; + const legacy = await this.request('GET', `/projects/${encodeURIComponent(legacyPath)}`); + if (legacy.status === 404) return null; + if (legacy.status !== 200) throw new Error(`Cannot verify legacy GitLab mirror (HTTP ${legacy.status})`); + const parent = await this.group(this.rootGroup); + if (legacy.data.path_with_namespace !== legacyPath || legacy.data.namespace?.id !== parent.id) { + throw new Error('Legacy GitLab mirror namespace verification failed'); + } + if (!await this.verifyIdentity(legacy.data, repo, true)) return null; + if (legacy.data.archived) throw new Error('Archived GitLab mirror requires a manual namespace transfer'); + await this.secure(legacy.data, repo); + const result = await this.request('PUT', `/projects/${legacy.data.id}/transfer`, {namespace: group.id}); + if (![200, 201, 202].includes(result.status)) { + throw new Error(`GitLab mirror transfer failed (HTTP ${result.status})`); + } + const desired = `${repo.targetGroup}/${repo.targetPath}`; + await this.waitForTransfer(legacy.data.id, group.id, desired); + return legacy.data.id; + } + + /** GitLab transfers can be asynchronous; verify the same project reached the requested namespace. */ + async waitForTransfer(projectId, namespaceId, path) { + for (let attempt = 0; attempt < 30; attempt++) { + const result = await this.request('GET', `/projects/${projectId}`); + if (result.status !== 200 || result.data.id !== projectId) throw new Error('GitLab transfer verification failed'); + if (result.data.namespace?.id === namespaceId && result.data.path_with_namespace === path) return; + await this.sleep(2000); + } + throw new Error('GitLab mirror transfer did not complete within one minute'); + } +} + +/** Prepare every repository independently so one permission or identity failure cannot stop other mirrors. */ +export async function prepareMirrors(repositories, options) { + const client = new GitLabMirrors(options); + await client.group(client.rootGroup); + const results = []; + await forEachSequential(repositories, async repo => { + try { results.push(await client.prepare(repo)); } + catch (error) { results.push({...repo, preparationError: error.message}); } + }); + return results; +} + +if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { + const path = process.argv[2]; + const prepared = await prepareMirrors(JSON.parse(readFileSync(path, 'utf8')), { + gitlabToken: process.env.GITLAB_TOKEN, githubToken: process.env.GITHUB_TOKEN, rootGroup: process.env.GITLAB_GROUP, + }); + writeFileSync(path, JSON.stringify(prepared), {mode: 0o600}); +} diff --git a/.github/scripts/mirror-gitlab.sh b/.github/scripts/mirror-gitlab.sh new file mode 100644 index 00000000..1e58679d --- /dev/null +++ b/.github/scripts/mirror-gitlab.sh @@ -0,0 +1,490 @@ +#!/usr/bin/env bash +set -euo pipefail + +if [[ -z "${GITHUB_TOKEN}" ]] || \ + [[ -z "${GITLAB_TOKEN}" ]] || \ + [[ -z "${GITLAB_SSH_PRIVATE_KEY}" ]]; then + echo "::error::GH_BOT_TOKEN, GITLAB_TOKEN, and GITLAB_SSH_PRIVATE_KEY must all be configured." + exit 1 +fi + +repository_file="${GITHUB_WORKSPACE}/repositories.json" +response_file="$(mktemp)" +temp_root="$(mktemp -d)" +trap 'rm -f "${response_file}" "${repository_file}"; rm -rf "${temp_root}"' EXIT + +ssh_dir="${temp_root}/ssh" +ssh_key="${ssh_dir}/id_ed25519" +ssh_known_hosts="${ssh_dir}/known_hosts" +mkdir -m 700 "${ssh_dir}" +printf '%s\n' "${GITLAB_SSH_PRIVATE_KEY}" > "${ssh_key}" +chmod 600 "${ssh_key}" +if ! ssh-keygen -y -P '' -f "${ssh_key}" > /dev/null; then + echo "::error::GITLAB_SSH_PRIVATE_KEY is not a valid unencrypted SSH private key." + exit 1 +fi + +# Published at https://docs.gitlab.com/user/gitlab_com/#ssh-known_hosts-entries. +gitlab_host_key='ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIAfuCHKVTjquxvt6CM6tdG4SLp1Btn/nOeHHE5UOzRdf' +printf 'gitlab.com %s\n' "${gitlab_host_key}" > "${ssh_known_hosts}" +chmod 600 "${ssh_known_hosts}" + +gitlab_host_fingerprint="$(ssh-keygen -lf "${ssh_known_hosts}" -E sha256 | awk '{print $2}')" +if [[ "${gitlab_host_fingerprint}" != \ + 'SHA256:eUXGGm1YGsMAS7vkcx6JOJdOGHPem5gQp4taiCfCLB8' ]]; then + echo "::error::The configured GitLab SSH host key has an unexpected fingerprint." + exit 1 +fi + +gitlab_ssh_args=( + -i "${ssh_key}" + -o BatchMode=yes + -o IdentitiesOnly=yes + -o StrictHostKeyChecking=yes + -o "UserKnownHostsFile=${ssh_known_hosts}" + -o ConnectTimeout=30 +) +printf -v gitlab_ssh_command '%q ' ssh "${gitlab_ssh_args[@]}" + +echo "Verifying GitLab SSH authentication..." +if ! ssh "${gitlab_ssh_args[@]}" -T git@gitlab.com; then + echo "::error::Unable to authenticate to GitLab with GITLAB_SSH_PRIVATE_KEY." + exit 1 +fi + +gitlab_request() { + local method="$1" + local url="$2" + local data="${3:-}" + local curl_args=( + --silent + --output "${response_file}" + --write-out '%{http_code}' + --request "${method}" + --header "Accept: application/json" + --header "PRIVATE-TOKEN: ${GITLAB_TOKEN}" + ) + + if [[ -n "${data}" ]]; then + curl_args+=( + --header "Content-Type: application/json" + --data "${data}" + ) + fi + + curl "${curl_args[@]}" "${url}" +} + +wait_for_gitlab_import() { + local project_id="$1" + local source_name="$2" + local deadline="$((SECONDS + 10800))" + local import_error + local import_status + local status + + while ((SECONDS < deadline)); do + if ! status="$(gitlab_request GET "${GITLAB_API_URL}/projects/${project_id}/import")"; then + echo "::error title=Mirror failed: ${source_name}::Unable to query the GitLab import." >&2 + return 1 + fi + if [[ "${status}" != "200" ]]; then + if [[ "${status}" == "403" ]]; then + echo "::error title=Mirror failed: ${source_name}::GITLAB_TOKEN needs Import API Read access." >&2 + return 1 + fi + echo >&2 \ + "::error title=Mirror failed: ${source_name}::Unable to query the GitLab import (HTTP ${status})." + return 1 + fi + + import_status="$(jq -er '.import_status' "${response_file}")" + case "${import_status}" in + finished) + echo "GitLab server-side import finished." + return 0 + ;; + scheduled | started) + echo "GitLab server-side import status: ${import_status}." + sleep 30 + ;; + failed) + import_error="$(jq -r '.import_error // "No import error detail was provided."' "${response_file}")" + import_error="${import_error//$'\r'/ }" + import_error="${import_error//$'\n'/ }" + echo "GitLab import failed: ${import_error}" + echo "::error title=Mirror failed: ${source_name}::GitLab server-side import failed." >&2 + return 1 + ;; + *) + echo >&2 \ + "::error title=Mirror failed: ${source_name}::Unexpected GitLab import status: ${import_status}." + return 1 + ;; + esac + done + + echo >&2 \ + "::error title=Mirror failed: ${source_name}::GitLab server-side import timed out after three hours." + return 1 +} + +node .github/scripts/gitlab-mirror.mjs "${repository_file}" + +github_auth="$(printf 'x-access-token:%s' "${GITHUB_TOKEN}" | base64 --wrap=0)" +echo "::add-mask::${github_auth}" +github_token_uri="$(jq -rn --arg value "${GITHUB_TOKEN}" '$value | @uri')" +echo "::add-mask::${github_token_uri}" + +# GitLab.com limits each Git push to 5 GiB. Seed large empty projects through GitLab's import API, +# using a lower threshold because GitHub's reported repository size is not the resulting pack size. +gitlab_import_threshold_kib="$((4 * 1024 * 1024))" + +repository_count="$(jq -er 'length' "${repository_file}")" +successful_repository_count=0 +failed_repository_names=() +failed_repository_reasons=() +for ((index = 0; index < repository_count; index++)); do + repository="$(jq -ec ".[${index}]" "${repository_file}")" + repository_number="$((index + 1))" + source_name="$(jq -er '.name' <<< "${repository}")" + source_clone_url="$(jq -er '.cloneUrl' <<< "${repository}")" + source_size_kib="$(jq -er '.sizeKiB' <<< "${repository}")" + target_name="$(jq -er '.targetName' <<< "${repository}")" + target_path="$(jq -er '.targetPath' <<< "${repository}")" + target_visibility="$(jq -er '.targetVisibility' <<< "${repository}")" + description="$(jq -er '.description' <<< "${repository}")" + group_id="$(jq -r '.namespaceId // empty' <<< "${repository}")" + target_group="$(jq -er '.targetGroup' <<< "${repository}")" + preparation_error="$(jq -r '.preparationError // empty' <<< "${repository}")" + expected_project_id="$(jq -r '.projectId // empty' <<< "${repository}")" + project_path="${target_group}/${target_path}" + encoded_project_path="$(jq -rn --arg value "${project_path}" '$value | @uri')" + repository_log="${temp_root}/repository-${index}.log" + + set +e + ( + set -euo pipefail + trap 'echo "::endgroup::"' EXIT + + echo "::group::[${repository_number}/${repository_count}] ${source_name} -> ${project_path}" + if [[ -n "${preparation_error}" ]]; then + echo "::error title=Mirror failed: ${source_name}::${preparation_error}" >&2 + exit 1 + fi + echo "Visibility: ${target_visibility}" + echo "Checking for the GitLab project..." + + project_created=false + gitlab_import_started=false + + if ! status="$(gitlab_request GET "${GITLAB_API_URL}/projects/${encoded_project_path}")"; then + echo "::error title=Mirror failed: ${source_name}::Unable to query the GitLab project." >&2 + exit 1 + fi + + if [[ "${status}" == "404" ]] || [[ "${status}" == "301" ]]; then + if [[ -n "${expected_project_id}" ]]; then + echo "::error title=Mirror failed: ${source_name}::Verified mirror path changed; refusing to create a replacement." >&2 + exit 1 + fi + if [[ "${status}" == "301" ]]; then + echo "The exact GitLab path redirects elsewhere; reclaiming ${project_path}." + else + echo "Project does not exist; creating ${project_path}..." + fi + + create_import_url="" + if ((source_size_kib >= gitlab_import_threshold_kib)); then + create_import_url="${source_clone_url}" + if [[ "${target_visibility}" == "private" ]]; then + create_import_url="https://x-access-token:${github_token_uri}@${source_clone_url#https://}" + echo "::add-mask::${create_import_url}" + fi + echo "The large repository will be imported as part of project creation." + fi + + payload="$( + jq -nc \ + --arg description "${description}" \ + --arg import_url "${create_import_url}" \ + --arg name "${target_name}" \ + --arg path "${target_path}" \ + --argjson namespace_id "${group_id}" \ + --arg visibility "${target_visibility}" \ + '{ + description: $description, + name: $name, + path: $path, + namespace_id: $namespace_id, + visibility: $visibility, + initialize_with_readme: false + } + if $import_url == "" then {} else {import_url: $import_url} end' + )" + if ! status="$(gitlab_request POST "${GITLAB_API_URL}/projects" "${payload}")"; then + echo "::error title=Mirror failed: ${source_name}::Unable to create the GitLab project." >&2 + exit 1 + fi + if [[ "${status}" != "201" ]]; then + create_error="$(jq -r '.message // "No response message was provided."' "${response_file}")" + create_error="${create_error//$'\r'/ }" + create_error="${create_error//$'\n'/ }" + echo "GitLab project creation response: ${create_error}" + error_message="Unable to create the GitLab project (HTTP ${status})." + echo "::error title=Mirror failed: ${source_name}::${error_message}" >&2 + exit 1 + fi + project_created=true + if [[ -n "${create_import_url}" ]]; then + gitlab_import_started=true + fi + elif [[ "${status}" != "200" ]]; then + echo >&2 \ + "::error title=Mirror failed: ${source_name}::Unable to query the GitLab project (HTTP ${status})." + exit 1 + else + if [[ -z "${expected_project_id}" ]] || \ + [[ "$(jq -er '.id' "${response_file}")" != "${expected_project_id}" ]]; then + echo "::error title=Mirror failed: ${source_name}::GitLab project identity changed after preparation." >&2 + exit 1 + fi + echo "Found existing GitLab project ${project_path}." + fi + + project_id="$(jq -er '.id' "${response_file}")" + if [[ "${project_created}" == "true" ]]; then + project_empty=true + else + project_empty="$(jq -r '.empty_repo == true' "${response_file}")" + fi + + # Make a private source private before changing metadata or pushing any Git data. + if [[ "${target_visibility}" == "private" ]]; then + echo "Securing ${project_path} as private before mirroring..." + privacy_payload="$(jq -nc '{visibility: "private"}')" + if ! status="$( + gitlab_request PUT "${GITLAB_API_URL}/projects/${project_id}" "${privacy_payload}" + )"; then + echo "::error title=Mirror failed: ${source_name}::Unable to secure the GitLab project." >&2 + exit 1 + fi + if [[ "${status}" != "200" ]] || \ + [[ "$(jq -er '.visibility' "${response_file}")" != "private" ]]; then + echo "::error title=Mirror failed: ${source_name}::GitLab privacy verification failed." >&2 + exit 1 + fi + fi + + echo "Updating and verifying the GitLab project metadata..." + payload="$( + jq -nc \ + --arg description "${description}" \ + --arg visibility "${target_visibility}" \ + '{description: $description, visibility: $visibility}' + )" + if ! status="$( + gitlab_request PUT "${GITLAB_API_URL}/projects/${project_id}" "${payload}" + )"; then + echo "::error title=Mirror failed: ${source_name}::Unable to update the GitLab project." >&2 + exit 1 + fi + if [[ "${status}" != "200" ]]; then + echo >&2 \ + "::error title=Mirror failed: ${source_name}::Unable to update the GitLab project (HTTP ${status})." + exit 1 + fi + + actual_description="$(jq -er '.description // ""' "${response_file}")" + actual_visibility="$(jq -er '.visibility' "${response_file}")" + if [[ "${actual_description}" != "${description}" ]]; then + echo "::error title=Mirror failed: ${source_name}::GitLab description verification failed." >&2 + exit 1 + fi + if [[ "${actual_visibility}" != "${target_visibility}" ]]; then + echo "::error title=Mirror failed: ${source_name}::GitLab visibility verification failed." >&2 + exit 1 + fi + + target_clone_url="$(jq -er '.ssh_url_to_repo' "${response_file}")" + + gitlab_incremental_seed=false + if [[ "${project_empty}" == "true" ]] && \ + ((source_size_kib >= gitlab_import_threshold_kib)) && \ + [[ "${gitlab_import_started}" != "true" ]]; then + echo "Empty large project detected (${source_size_kib} KiB reported by GitHub)." + echo "Scheduling a GitLab server-side Git import to avoid the 5 GiB push limit..." + if [[ "${target_visibility}" == "private" ]]; then + import_payload="$( + jq -nc \ + --arg import_url "${source_clone_url}" \ + --arg import_url_user "x-access-token" \ + --arg import_url_password "${GITHUB_TOKEN}" \ + '{ + import_url: $import_url, + import_url_user: $import_url_user, + import_url_password: $import_url_password + }' + )" + else + import_payload="$( + jq -nc \ + --arg import_url "${source_clone_url}" \ + '{import_url: $import_url}' + )" + fi + + if ! status="$( + gitlab_request POST "${GITLAB_API_URL}/projects/${project_id}/import/git" "${import_payload}" + )"; then + echo "::error title=Mirror failed: ${source_name}::Unable to schedule the GitLab import." >&2 + exit 1 + fi + if [[ "${status}" == "403" ]]; then + echo "::error title=Mirror failed: ${source_name}::GITLAB_TOKEN needs Import API Create access." >&2 + exit 1 + fi + + import_response_message="$(jq -r '.message // "No response message was provided."' "${response_file}")" + import_response_message="${import_response_message//$'\r'/ }" + import_response_message="${import_response_message//$'\n'/ }" + if [[ "${status}" == "409" ]]; then + case "${import_response_message}" in + "Import already in progress") + echo "A GitLab import is already in progress; continuing to monitor it." + gitlab_import_started=true + ;; + "Project already has a repository") + echo "GitLab has repository storage from an earlier push; using incremental seeding." + gitlab_incremental_seed=true + ;; + *) + echo "GitLab import conflict: ${import_response_message}" + echo "::error title=Mirror failed: ${source_name}::Unable to schedule the GitLab import." >&2 + exit 1 + ;; + esac + elif [[ "${status}" != "201" ]] && [[ "${status}" != "202" ]]; then + echo "GitLab import response: ${import_response_message}" + echo "::error title=Mirror failed: ${source_name}::Unable to schedule the GitLab import" \ + "(HTTP ${status})." >&2 + exit 1 + else + gitlab_import_started=true + fi + elif [[ "${project_empty}" != "true" ]] && \ + ((source_size_kib >= gitlab_import_threshold_kib)); then + echo "Large project is already populated; using an incremental Git push." + fi + + mirror_dir="${temp_root}/repository.git" + echo "Cloning ${source_clone_url} as a mirror..." + if ! git \ + -c http.https://github.com/.extraheader="AUTHORIZATION: basic ${github_auth}" \ + clone --mirror --progress "${source_clone_url}" "${mirror_dir}"; then + echo "::error title=Mirror failed: ${source_name}::Unable to clone the GitHub repository." >&2 + exit 1 + fi + + mapfile -t pull_refs < <( + git -C "${mirror_dir}" for-each-ref --format='%(refname)' refs/pull/ + ) + if ((${#pull_refs[@]} > 0)); then + echo "Removing ${#pull_refs[@]} GitHub-only pull request refs from the mirror..." + printf 'delete %s\n' "${pull_refs[@]}" | git -C "${mirror_dir}" update-ref --stdin + fi + + if [[ "${gitlab_import_started}" == "true" ]]; then + echo "Waiting for the GitLab server-side import to finish..." + if ! wait_for_gitlab_import "${project_id}" "${source_name}"; then + exit 1 + fi + fi + + mapfile -t mirror_refs < <( + git -C "${mirror_dir}" for-each-ref --format='%(refname)' + ) + if ((${#mirror_refs[@]} == 0)); then + echo "The GitHub repository has no refs to mirror." + if [[ "${project_empty}" != "true" ]]; then + echo "::error title=Mirror failed: ${source_name}::The GitHub repository is empty, but" \ + "the GitLab repository is not. Refusing to leave stale GitLab refs." >&2 + exit 1 + fi + echo "The GitLab repository is also empty; no Git push is required." + else + if [[ "${gitlab_incremental_seed}" == "true" ]]; then + mapfile -t seed_refs < <( + git -C "${mirror_dir}" for-each-ref --format='%(refname)' refs/heads/ refs/tags/ + ) + echo "Seeding ${#seed_refs[@]} branches and tags in separate pushes..." + for seed_ref in "${seed_refs[@]}"; do + echo "Seeding ${seed_ref}..." + if ! git \ + -C "${mirror_dir}" \ + -c core.sshCommand="${gitlab_ssh_command}" \ + push --progress "${target_clone_url}" "+${seed_ref}:${seed_ref}"; then + echo "::error title=Mirror failed: ${source_name}::Unable to seed ${seed_ref} on GitLab." >&2 + exit 1 + fi + done + fi + + echo "Pushing the mirror to ${target_clone_url}..." + if ! git \ + -C "${mirror_dir}" \ + -c core.sshCommand="${gitlab_ssh_command}" \ + push --mirror --progress "${target_clone_url}"; then + echo "::error title=Mirror failed: ${source_name}::Unable to push the GitLab mirror." >&2 + exit 1 + fi + fi + rm -rf "${mirror_dir}" + echo "Completed mirror for ${source_name}." + ) 2>&1 | tee "${repository_log}" + repository_status="${PIPESTATUS[0]}" + set -e + + rm -rf "${temp_root}/repository.git" + if ((repository_status == 0)); then + successful_repository_count="$((successful_repository_count + 1))" + else + error_line="$(grep -E '^::error(::| )' "${repository_log}" | tail -n 1 || true)" + if [[ -n "${error_line}" ]]; then + failure_reason="${error_line##*::}" + else + failure_reason="Repository mirroring failed; inspect its log group." + fi + failure_reason="${failure_reason//$'\r'/ }" + failure_reason="${failure_reason//$'\n'/ }" + failed_repository_names+=("${source_name}") + failed_repository_reasons+=("${failure_reason}") + echo "Recorded mirror failure for ${source_name}; continuing with the next repository." + fi + rm -f "${repository_log}" +done + +failed_repository_count="${#failed_repository_names[@]}" +{ + echo "## GitHub to GitLab mirror summary" + echo + echo "- Successful: ${successful_repository_count}" + echo "- Failed: ${failed_repository_count}" + if ((failed_repository_count > 0)); then + echo + echo "| Repository | Failure |" + echo "| --- | --- |" + for ((failure_index = 0; failure_index < failed_repository_count; failure_index++)); do + summary_name="${failed_repository_names[$failure_index]//|/\\|}" + summary_reason="${failed_repository_reasons[$failure_index]//|/\\|}" + printf '| %s | %s |\n' "\`${summary_name}\`" "${summary_reason}" + done + fi +} >> "${GITHUB_STEP_SUMMARY}" + +if ((failed_repository_count > 0)); then + echo "::error::Failed to mirror ${failed_repository_count} of ${repository_count} repositories." \ + "See the step summary." + exit 1 +fi + +echo "Mirrored all ${repository_count} repositories to GitLab." diff --git a/.github/scripts/organization-repositories.mjs b/.github/scripts/organization-repositories.mjs new file mode 100644 index 00000000..de6fadb6 --- /dev/null +++ b/.github/scripts/organization-repositories.mjs @@ -0,0 +1,85 @@ +import {readFileSync} from 'node:fs'; + +import {forEachSequential} from './async-iteration.mjs'; + +/** Read and validate the shared organization registry, without maintaining another org list. */ +export function readOrganizations(path = 'organizations.json') { + const organizations = JSON.parse(readFileSync(path, 'utf8')); + if (!Array.isArray(organizations) || !organizations.length || organizations.some(name => + typeof name !== 'string' || !/^[a-z\d][a-z\d-]*$/i.test(name)) || + new Set(organizations.map(name => name.toLowerCase())).size !== organizations.length) { + throw new Error('Invalid organization registry'); + } + return organizations; +} + +/** Discover canonical repository identities once per org, retaining caller-specific eligibility rules. */ +export async function discoverRepositories(github, organizations, { + includeArchived = false, includeForks = true, includePrivate = false, +} = {}) { + const repositories = new Map(); + await forEachSequential(organizations, async organization => { + const options = github.rest.repos.listForOrg.endpoint.merge({org: organization, per_page: 100}); + const repos = await github.paginate(options); + for (const repo of repos) { + if (!Number.isSafeInteger(repo.id) || repo.id <= 0 || + repo.owner?.login?.toLowerCase() !== organization.toLowerCase() || + !/^[\w.-]+$/.test(repo.name) || ['.', '..'].includes(repo.name) || + repo.full_name !== `${repo.owner.login}/${repo.name}`) { + throw new Error(`Invalid repository identity returned for ${organization}`); + } + const previous = repositories.get(repo.id); + if (previous && previous.full_name !== repo.full_name) { + throw new Error('Repository ownership changed during discovery; retry the workflow'); + } + if ((!includeArchived && repo.archived) || (!includeForks && repo.fork) || + (!includePrivate && repo.visibility !== 'public')) continue; + repositories.set(repo.id, repo); + } + }); + return [...repositories.values()]; +} + +/** Build public maintenance matrices; private mirrors are handled within the single mirror job. */ +export async function maintenanceMatrix(github, organizations, {profile, includeForks = true, issuesOnly = false} = {}) { + const repos = await discoverRepositories(github, organizations, {includeForks}); + const include = []; + await forEachSequential(repos, async repo => { + if (issuesOnly && repo.has_issues === false) return; + if (profile) { + const {data: values} = await github.request('GET /repos/{owner}/{repo}/properties/values', { + owner: repo.owner.login, repo: repo.name, + }); + const profiles = values.find(property => property.property_name === 'workflow_profiles')?.value ?? []; + if (!Array.isArray(profiles) || !profiles.every(value => typeof value === 'string')) { + throw new Error(`Invalid workflow_profiles custom property for ${repo.full_name}`); + } + if (!profiles.includes(profile)) return; + } + include.push({owner: repo.owner.login, repo: repo.name, full_name: repo.full_name}); + }); + return {include}; +} + +/** Resolve a qualified name, or retain a short-name entry point when it is unambiguous. */ +export function resolveRepository(repositories, selector) { + const normalized = selector.trim().toLowerCase(); + const matches = repositories.filter(repo => + (normalized.includes('/') ? repo.full_name : repo.name).toLowerCase() === normalized); + if (matches.length !== 1) { + throw new Error('Repository must identify exactly one eligible repository in organizations.json; use owner/repository'); + } + return matches[0]; +} + +/** Keep shared actions on master for every registered org while pinning third-party actions. */ +export function pinactConfiguration(organizations) { + return JSON.stringify({ + version: 3, + ignore_actions: [ + ...organizations.map(name => ({name: `${name}/.*`, ref: 'master'})), + {name: 'Homebrew/actions/.*', ref: 'main'}, + ], + separator: ' # ', + }, null, 2); +} diff --git a/.github/workflows/README.md b/.github/workflows/README.md index 6bc710af..baf0fad2 100644 --- a/.github/workflows/README.md +++ b/.github/workflows/README.md @@ -33,7 +33,36 @@ requires `GH_TOKEN` with `write:packages` access to the existing LizardByte pack passes `GH_BOT_TOKEN`. npmjs trusted-publisher bindings and external service installations must also be updated for the destination owner when transferring a repository. -Repositories excluded from the GitLab mirror are listed in `.github/repositories-to-ignore.json`. +Central stale-issue, license-year, CPM, Pinact, and build-deps workflows read `organizations.json` as well. +Repository matrices use qualified identities so matching names in different orgs remain distinct. Public +maintenance matrices exclude private and archived repositories; license updates also exclude forks. CPM +continues to select the `cpm` custom-property profile. Manual stale/CPM dispatches preview their matrices. +The build-deps updater retains its existing project choices and resolves their current owners through the +registry; dependency releases still come from `LizardByte/build-deps`. Pinact accepts qualified repo names +or unambiguous short names and preserves shared `master` references across all registered orgs. + +The GitLab mirror reads all registered orgs, including private, archived, and forked source repositories. +Repositories excluded from it are listed as qualified `owner/repository` names in +`.github/repositories-to-ignore.json`. The GitLab mirror also excludes GitHub temporary security advisory forks matching `-ghsa-xxxx-xxxx-xxxx`. + +GitHub repositories in LizardByte retain their existing GitLab paths under `lizardbyte`. Other GitHub orgs +use automatically created subgroups: for example, `SunshineStream/Sunshine` maps to +`lizardbyte/sunshinestream/Sunshine`. The existing parent group must exist. Subgroups inherit its visibility +and membership; the [API cannot create top-level groups on GitLab.com](https://docs.gitlab.com/api/groups/#create-a-group). + +When a GitHub repository moves, the mirror verifies the existing root project's GitHub repository ID and +transfers that GitLab project into the matching subgroup. Legacy descriptions are checked through GitHub's +repository API, following transfer redirects. GitLab project IDs, Git history, issues, and settings are retained. +New mirror descriptions record the immutable GitHub ID. Identity mismatches at the destination, lookup failures, +and incomplete transfers stop that repository's delivery while other repositories continue. Private sources are +verified as private before any transfer or Git push. Existing archived GitLab mirrors require manual handling; +the workflow does not unarchive them. Large-project imports and the existing GitLab SSH transport remain in use. + +`GITLAB_TOKEN` needs access to read/create subgroups under `lizardbyte`, create/update projects and their visibility, +transfer existing mirror projects, and use the existing import APIs. Transfers require project Owner access and +permission to create projects in the destination group. See [GitLab project transfers](https://docs.gitlab.com/user/project/working_with_projects/#transfer-a-project). +The SSH identity must retain Git push access inherited +from the parent group. These use the existing controller credentials; no destination-org secrets are needed. diff --git a/.github/workflows/__issues-stale.yml b/.github/workflows/__issues-stale.yml index 31213a3b..11a8c41a 100644 --- a/.github/workflows/__issues-stale.yml +++ b/.github/workflows/__issues-stale.yml @@ -19,29 +19,29 @@ jobs: runs-on: ubuntu-latest outputs: matrix: ${{ steps.set-matrix.outputs.result }} - permissions: {} + has-repositories: ${{ steps.set-matrix.outputs.has-repositories }} + permissions: + contents: read steps: - # get all repos in this org + - name: Checkout controller + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - name: Get repos id: set-matrix uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 with: + github-token: ${{ secrets.GH_BOT_TOKEN }} script: | - const opts = github.rest.repos.listForOrg.endpoint.merge({ org: context.repo.owner }); - const repos = await github.paginate(opts) - - // create a GitHub strategy matrix - let matrix = { "include": [] }; - for (const repo of repos) { - if (!repo.archived) { - matrix.include.push({ "repo": repo.name }); - } - } - return matrix + const {readOrganizations, maintenanceMatrix} = await import( + `${process.env.GITHUB_WORKSPACE}/.github/scripts/organization-repositories.mjs` + ); + const matrix = await maintenanceMatrix(github, readOrganizations(), {issuesOnly: true}); + core.setOutput('has-repositories', matrix.include.length > 0); + return matrix; test-matrix: - name: Test Matrix - ${{ matrix.repo }} - if: github.event_name == 'workflow_dispatch' + name: Test Matrix - ${{ matrix.full_name }} + if: github.event_name == 'workflow_dispatch' && needs.setup-matrix.outputs.has-repositories == 'true' needs: setup-matrix permissions: {} runs-on: ubuntu-latest @@ -54,7 +54,7 @@ jobs: stale: name: Check Stale Issues / PRs - if: github.event_name == 'schedule' + if: github.event_name == 'schedule' && needs.setup-matrix.outputs.has-repositories == 'true' needs: setup-matrix permissions: {} runs-on: ubuntu-latest @@ -77,7 +77,7 @@ jobs: exempt-all-assignees: true exempt-issue-labels: 'planned' exempt-pr-labels: 'dependencies,l10n' - repo-owner: ${{ github.repository_owner }} + repo-owner: ${{ matrix.owner }} repo-name: ${{ matrix.repo }} stale-issue-label: 'stale' stale-issue-message: > diff --git a/.github/workflows/__mirror-github-to-gitlab.yml b/.github/workflows/__mirror-github-to-gitlab.yml index 24e3fb9c..c25777e9 100644 --- a/.github/workflows/__mirror-github-to-gitlab.yml +++ b/.github/workflows/__mirror-github-to-gitlab.yml @@ -1,5 +1,5 @@ --- -# Mirror all repositories in the LizardByte GitHub organization to GitLab. +# Mirror registered GitHub organizations into the matching GitLab namespace. name: Mirror GitHub to GitLab permissions: {} @@ -24,70 +24,30 @@ jobs: - name: Checkout repository uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - name: Setup Node + uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 + with: + node-version: '26' + - name: Get repositories uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 with: github-token: ${{ secrets.GH_BOT_TOKEN }} script: | const fs = require('fs'); - const ignoredRepositoryNames = JSON.parse( - fs.readFileSync('.github/repositories-to-ignore.json', 'utf8'), - ); - if ( - !Array.isArray(ignoredRepositoryNames) - || !ignoredRepositoryNames.every((name) => typeof name === 'string' && name.length > 0) - ) { - throw new Error('Expected an array of non-empty repository names.'); - } - const ignoredRepositories = new Set(ignoredRepositoryNames); - const securityAdvisoryIdSegment = '[23456789cfghjmpqrvwx]{4}'; - const securityAdvisoryForkPattern = new RegExp( - `-ghsa-${securityAdvisoryIdSegment}-${securityAdvisoryIdSegment}-${securityAdvisoryIdSegment}$`, - 'i', + const {readOrganizations, discoverRepositories} = await import( + `${process.env.GITHUB_WORKSPACE}/.github/scripts/organization-repositories.mjs` ); - const shouldIgnoreRepository = (repo) => ( - ignoredRepositories.has(repo.name) || securityAdvisoryForkPattern.test(repo.name) + const {mirrorRepositories} = await import( + `${process.env.GITHUB_WORKSPACE}/.github/scripts/gitlab-mirror.mjs` ); - const opts = github.rest.repos.listForOrg.endpoint.merge({ org: context.repo.owner }); - const repos = await github.paginate(opts); - const gitlabTarget = (repo) => { - const prefixNames = { '.': 'dot-', '-': 'dash-', '_': 'underscore-' }; - const lowerName = repo.name.toLowerCase(); - let targetName = lowerName.replace( - /^[._-]+/, - (prefix) => [...prefix].map((character) => prefixNames[character]).join(''), - ); - targetName = targetName - .replace(/[^a-z0-9_.-]+/g, '-') - .replace(/[._-]{2,}/g, '-') - .replace(/[._-]+$/g, ''); - - if (targetName.endsWith('.git') || targetName.endsWith('.atom')) { - targetName += '-repo'; - } - if (!targetName) { - targetName = 'repository'; - } - - const transformed = targetName !== lowerName; - return { - targetName: transformed ? targetName : repo.name, - targetPath: transformed ? `${targetName}-${repo.id}` : repo.name, - }; - }; - const repositoryData = repos - .filter((repo) => !shouldIgnoreRepository(repo)) - .map((repo) => ({ - cloneUrl: repo.clone_url, - name: repo.name, - sizeKiB: repo.size, - targetVisibility: repo.visibility === 'public' ? 'public' : 'private', - ...gitlabTarget(repo), - })); - - fs.writeFileSync('repositories.json', JSON.stringify(repositoryData), { mode: 0o600 }); - core.info(`Ignored ${repos.length - repositoryData.length} repositories.`); - core.info(`Prepared ${repositoryData.length} repositories for mirroring.`); + const ignored = JSON.parse(fs.readFileSync('.github/repositories-to-ignore.json', 'utf8')); + const repos = await discoverRepositories(github, readOrganizations(), { + includeArchived: true, includeForks: true, includePrivate: true, + }); + const targets = mirrorRepositories(repos, ignored, 'lizardbyte', context.repo.owner); + fs.writeFileSync('repositories.json', JSON.stringify(targets), {mode: 0o600}); + core.info(`Prepared ${targets.length} repositories for mirroring.`); - name: Mirror repositories shell: bash @@ -98,485 +58,4 @@ jobs: GITLAB_GROUP: lizardbyte GITLAB_SSH_PRIVATE_KEY: ${{ secrets.GITLAB_SSH_PRIVATE_KEY }} GITLAB_TOKEN: ${{ secrets.GITLAB_TOKEN }} - run: | - set -euo pipefail - - if [[ -z "${GITHUB_TOKEN}" ]] || \ - [[ -z "${GITLAB_TOKEN}" ]] || \ - [[ -z "${GITLAB_SSH_PRIVATE_KEY}" ]]; then - echo "::error::GH_BOT_TOKEN, GITLAB_TOKEN, and GITLAB_SSH_PRIVATE_KEY must all be configured." - exit 1 - fi - - repository_file="${GITHUB_WORKSPACE}/repositories.json" - response_file="$(mktemp)" - temp_root="$(mktemp -d)" - trap 'rm -f "${response_file}" "${repository_file}"; rm -rf "${temp_root}"' EXIT - - ssh_dir="${temp_root}/ssh" - ssh_key="${ssh_dir}/id_ed25519" - ssh_known_hosts="${ssh_dir}/known_hosts" - mkdir -m 700 "${ssh_dir}" - printf '%s\n' "${GITLAB_SSH_PRIVATE_KEY}" > "${ssh_key}" - chmod 600 "${ssh_key}" - if ! ssh-keygen -y -P '' -f "${ssh_key}" > /dev/null; then - echo "::error::GITLAB_SSH_PRIVATE_KEY is not a valid unencrypted SSH private key." - exit 1 - fi - - # Published at https://docs.gitlab.com/user/gitlab_com/#ssh-known_hosts-entries. - gitlab_host_key='ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIAfuCHKVTjquxvt6CM6tdG4SLp1Btn/nOeHHE5UOzRdf' - printf 'gitlab.com %s\n' "${gitlab_host_key}" > "${ssh_known_hosts}" - chmod 600 "${ssh_known_hosts}" - - gitlab_host_fingerprint="$(ssh-keygen -lf "${ssh_known_hosts}" -E sha256 | awk '{print $2}')" - if [[ "${gitlab_host_fingerprint}" != \ - 'SHA256:eUXGGm1YGsMAS7vkcx6JOJdOGHPem5gQp4taiCfCLB8' ]]; then - echo "::error::The configured GitLab SSH host key has an unexpected fingerprint." - exit 1 - fi - - gitlab_ssh_args=( - -i "${ssh_key}" - -o BatchMode=yes - -o IdentitiesOnly=yes - -o StrictHostKeyChecking=yes - -o "UserKnownHostsFile=${ssh_known_hosts}" - -o ConnectTimeout=30 - ) - printf -v gitlab_ssh_command '%q ' ssh "${gitlab_ssh_args[@]}" - - echo "Verifying GitLab SSH authentication..." - if ! ssh "${gitlab_ssh_args[@]}" -T git@gitlab.com; then - echo "::error::Unable to authenticate to GitLab with GITLAB_SSH_PRIVATE_KEY." - exit 1 - fi - - gitlab_request() { - local method="$1" - local url="$2" - local data="${3:-}" - local curl_args=( - --silent - --output "${response_file}" - --write-out '%{http_code}' - --request "${method}" - --header "Accept: application/json" - --header "PRIVATE-TOKEN: ${GITLAB_TOKEN}" - ) - - if [[ -n "${data}" ]]; then - curl_args+=( - --header "Content-Type: application/json" - --data "${data}" - ) - fi - - curl "${curl_args[@]}" "${url}" - } - - wait_for_gitlab_import() { - local project_id="$1" - local source_name="$2" - local deadline="$((SECONDS + 10800))" - local import_error - local import_status - local status - - while ((SECONDS < deadline)); do - if ! status="$(gitlab_request GET "${GITLAB_API_URL}/projects/${project_id}/import")"; then - echo "::error title=Mirror failed: ${source_name}::Unable to query the GitLab import." >&2 - return 1 - fi - if [[ "${status}" != "200" ]]; then - if [[ "${status}" == "403" ]]; then - echo "::error title=Mirror failed: ${source_name}::GITLAB_TOKEN needs Import API Read access." >&2 - return 1 - fi - echo >&2 \ - "::error title=Mirror failed: ${source_name}::Unable to query the GitLab import (HTTP ${status})." - return 1 - fi - - import_status="$(jq -er '.import_status' "${response_file}")" - case "${import_status}" in - finished) - echo "GitLab server-side import finished." - return 0 - ;; - scheduled | started) - echo "GitLab server-side import status: ${import_status}." - sleep 30 - ;; - failed) - import_error="$(jq -r '.import_error // "No import error detail was provided."' "${response_file}")" - import_error="${import_error//$'\r'/ }" - import_error="${import_error//$'\n'/ }" - echo "GitLab import failed: ${import_error}" - echo "::error title=Mirror failed: ${source_name}::GitLab server-side import failed." >&2 - return 1 - ;; - *) - echo >&2 \ - "::error title=Mirror failed: ${source_name}::Unexpected GitLab import status: ${import_status}." - return 1 - ;; - esac - done - - echo >&2 \ - "::error title=Mirror failed: ${source_name}::GitLab server-side import timed out after three hours." - return 1 - } - - encoded_group="$(jq -rn --arg value "${GITLAB_GROUP}" '$value | @uri')" - if ! status="$(gitlab_request GET "${GITLAB_API_URL}/groups/${encoded_group}")"; then - echo "::error::Unable to query the GitLab group." - exit 1 - fi - if [[ "${status}" != "200" ]]; then - echo "::error::Unable to query the GitLab group (HTTP ${status})." - exit 1 - fi - group_id="$(jq -er '.id' "${response_file}")" - - github_auth="$(printf 'x-access-token:%s' "${GITHUB_TOKEN}" | base64 --wrap=0)" - echo "::add-mask::${github_auth}" - github_token_uri="$(jq -rn --arg value "${GITHUB_TOKEN}" '$value | @uri')" - echo "::add-mask::${github_token_uri}" - - # GitLab.com limits each Git push to 5 GiB. Seed large empty projects through GitLab's import API, - # using a lower threshold because GitHub's reported repository size is not the resulting pack size. - gitlab_import_threshold_kib="$((4 * 1024 * 1024))" - - repository_count="$(jq -er 'length' "${repository_file}")" - successful_repository_count=0 - failed_repository_names=() - failed_repository_reasons=() - for ((index = 0; index < repository_count; index++)); do - repository="$(jq -ec ".[${index}]" "${repository_file}")" - repository_number="$((index + 1))" - source_name="$(jq -er '.name' <<< "${repository}")" - source_clone_url="$(jq -er '.cloneUrl' <<< "${repository}")" - source_size_kib="$(jq -er '.sizeKiB' <<< "${repository}")" - target_name="$(jq -er '.targetName' <<< "${repository}")" - target_path="$(jq -er '.targetPath' <<< "${repository}")" - target_visibility="$(jq -er '.targetVisibility' <<< "${repository}")" - description="Mirror of ${source_clone_url}" - project_path="${GITLAB_GROUP}/${target_path}" - encoded_project_path="$(jq -rn --arg value "${project_path}" '$value | @uri')" - repository_log="${temp_root}/repository-${index}.log" - - set +e - ( - set -euo pipefail - trap 'echo "::endgroup::"' EXIT - - echo "::group::[${repository_number}/${repository_count}] ${source_name} -> ${project_path}" - echo "Visibility: ${target_visibility}" - echo "Checking for the GitLab project..." - - project_created=false - gitlab_import_started=false - - if ! status="$(gitlab_request GET "${GITLAB_API_URL}/projects/${encoded_project_path}")"; then - echo "::error title=Mirror failed: ${source_name}::Unable to query the GitLab project." >&2 - exit 1 - fi - - if [[ "${status}" == "404" ]] || [[ "${status}" == "301" ]]; then - if [[ "${status}" == "301" ]]; then - echo "The exact GitLab path redirects elsewhere; reclaiming ${project_path}." - else - echo "Project does not exist; creating ${project_path}..." - fi - - create_import_url="" - if ((source_size_kib >= gitlab_import_threshold_kib)); then - create_import_url="${source_clone_url}" - if [[ "${target_visibility}" == "private" ]]; then - create_import_url="https://x-access-token:${github_token_uri}@${source_clone_url#https://}" - echo "::add-mask::${create_import_url}" - fi - echo "The large repository will be imported as part of project creation." - fi - - payload="$( - jq -nc \ - --arg description "${description}" \ - --arg import_url "${create_import_url}" \ - --arg name "${target_name}" \ - --arg path "${target_path}" \ - --argjson namespace_id "${group_id}" \ - --arg visibility "${target_visibility}" \ - '{ - description: $description, - name: $name, - path: $path, - namespace_id: $namespace_id, - visibility: $visibility, - initialize_with_readme: false - } + if $import_url == "" then {} else {import_url: $import_url} end' - )" - if ! status="$(gitlab_request POST "${GITLAB_API_URL}/projects" "${payload}")"; then - echo "::error title=Mirror failed: ${source_name}::Unable to create the GitLab project." >&2 - exit 1 - fi - if [[ "${status}" != "201" ]]; then - create_error="$(jq -r '.message // "No response message was provided."' "${response_file}")" - create_error="${create_error//$'\r'/ }" - create_error="${create_error//$'\n'/ }" - echo "GitLab project creation response: ${create_error}" - error_message="Unable to create the GitLab project (HTTP ${status})." - echo "::error title=Mirror failed: ${source_name}::${error_message}" >&2 - exit 1 - fi - project_created=true - if [[ -n "${create_import_url}" ]]; then - gitlab_import_started=true - fi - elif [[ "${status}" != "200" ]]; then - echo >&2 \ - "::error title=Mirror failed: ${source_name}::Unable to query the GitLab project (HTTP ${status})." - exit 1 - else - echo "Found existing GitLab project ${project_path}." - fi - - project_id="$(jq -er '.id' "${response_file}")" - if [[ "${project_created}" == "true" ]]; then - project_empty=true - else - project_empty="$(jq -r '.empty_repo == true' "${response_file}")" - fi - - # Make a private source private before changing metadata or pushing any Git data. - if [[ "${target_visibility}" == "private" ]]; then - echo "Securing ${project_path} as private before mirroring..." - privacy_payload="$(jq -nc '{visibility: "private"}')" - if ! status="$( - gitlab_request PUT "${GITLAB_API_URL}/projects/${project_id}" "${privacy_payload}" - )"; then - echo "::error title=Mirror failed: ${source_name}::Unable to secure the GitLab project." >&2 - exit 1 - fi - if [[ "${status}" != "200" ]] || \ - [[ "$(jq -er '.visibility' "${response_file}")" != "private" ]]; then - echo "::error title=Mirror failed: ${source_name}::GitLab privacy verification failed." >&2 - exit 1 - fi - fi - - echo "Updating and verifying the GitLab project metadata..." - payload="$( - jq -nc \ - --arg description "${description}" \ - --arg visibility "${target_visibility}" \ - '{description: $description, visibility: $visibility}' - )" - if ! status="$( - gitlab_request PUT "${GITLAB_API_URL}/projects/${project_id}" "${payload}" - )"; then - echo "::error title=Mirror failed: ${source_name}::Unable to update the GitLab project." >&2 - exit 1 - fi - if [[ "${status}" != "200" ]]; then - echo >&2 \ - "::error title=Mirror failed: ${source_name}::Unable to update the GitLab project (HTTP ${status})." - exit 1 - fi - - actual_description="$(jq -er '.description // ""' "${response_file}")" - actual_visibility="$(jq -er '.visibility' "${response_file}")" - if [[ "${actual_description}" != "${description}" ]]; then - echo "::error title=Mirror failed: ${source_name}::GitLab description verification failed." >&2 - exit 1 - fi - if [[ "${actual_visibility}" != "${target_visibility}" ]]; then - echo "::error title=Mirror failed: ${source_name}::GitLab visibility verification failed." >&2 - exit 1 - fi - - target_clone_url="$(jq -er '.ssh_url_to_repo' "${response_file}")" - - gitlab_incremental_seed=false - if [[ "${project_empty}" == "true" ]] && \ - ((source_size_kib >= gitlab_import_threshold_kib)) && \ - [[ "${gitlab_import_started}" != "true" ]]; then - echo "Empty large project detected (${source_size_kib} KiB reported by GitHub)." - echo "Scheduling a GitLab server-side Git import to avoid the 5 GiB push limit..." - if [[ "${target_visibility}" == "private" ]]; then - import_payload="$( - jq -nc \ - --arg import_url "${source_clone_url}" \ - --arg import_url_user "x-access-token" \ - --arg import_url_password "${GITHUB_TOKEN}" \ - '{ - import_url: $import_url, - import_url_user: $import_url_user, - import_url_password: $import_url_password - }' - )" - else - import_payload="$( - jq -nc \ - --arg import_url "${source_clone_url}" \ - '{import_url: $import_url}' - )" - fi - - if ! status="$( - gitlab_request POST "${GITLAB_API_URL}/projects/${project_id}/import/git" "${import_payload}" - )"; then - echo "::error title=Mirror failed: ${source_name}::Unable to schedule the GitLab import." >&2 - exit 1 - fi - if [[ "${status}" == "403" ]]; then - echo "::error title=Mirror failed: ${source_name}::GITLAB_TOKEN needs Import API Create access." >&2 - exit 1 - fi - - import_response_message="$(jq -r '.message // "No response message was provided."' "${response_file}")" - import_response_message="${import_response_message//$'\r'/ }" - import_response_message="${import_response_message//$'\n'/ }" - if [[ "${status}" == "409" ]]; then - case "${import_response_message}" in - "Import already in progress") - echo "A GitLab import is already in progress; continuing to monitor it." - gitlab_import_started=true - ;; - "Project already has a repository") - echo "GitLab has repository storage from an earlier push; using incremental seeding." - gitlab_incremental_seed=true - ;; - *) - echo "GitLab import conflict: ${import_response_message}" - echo "::error title=Mirror failed: ${source_name}::Unable to schedule the GitLab import." >&2 - exit 1 - ;; - esac - elif [[ "${status}" != "201" ]] && [[ "${status}" != "202" ]]; then - echo "GitLab import response: ${import_response_message}" - echo "::error title=Mirror failed: ${source_name}::Unable to schedule the GitLab import" \ - "(HTTP ${status})." >&2 - exit 1 - else - gitlab_import_started=true - fi - elif [[ "${project_empty}" != "true" ]] && \ - ((source_size_kib >= gitlab_import_threshold_kib)); then - echo "Large project is already populated; using an incremental Git push." - fi - - mirror_dir="${temp_root}/repository.git" - echo "Cloning ${source_clone_url} as a mirror..." - if ! git \ - -c http.https://github.com/.extraheader="AUTHORIZATION: basic ${github_auth}" \ - clone --mirror --progress "${source_clone_url}" "${mirror_dir}"; then - echo "::error title=Mirror failed: ${source_name}::Unable to clone the GitHub repository." >&2 - exit 1 - fi - - mapfile -t pull_refs < <( - git -C "${mirror_dir}" for-each-ref --format='%(refname)' refs/pull/ - ) - if ((${#pull_refs[@]} > 0)); then - echo "Removing ${#pull_refs[@]} GitHub-only pull request refs from the mirror..." - printf 'delete %s\n' "${pull_refs[@]}" | git -C "${mirror_dir}" update-ref --stdin - fi - - if [[ "${gitlab_import_started}" == "true" ]]; then - echo "Waiting for the GitLab server-side import to finish..." - if ! wait_for_gitlab_import "${project_id}" "${source_name}"; then - exit 1 - fi - fi - - mapfile -t mirror_refs < <( - git -C "${mirror_dir}" for-each-ref --format='%(refname)' - ) - if ((${#mirror_refs[@]} == 0)); then - echo "The GitHub repository has no refs to mirror." - if [[ "${project_empty}" != "true" ]]; then - echo "::error title=Mirror failed: ${source_name}::The GitHub repository is empty, but" \ - "the GitLab repository is not. Refusing to leave stale GitLab refs." >&2 - exit 1 - fi - echo "The GitLab repository is also empty; no Git push is required." - else - if [[ "${gitlab_incremental_seed}" == "true" ]]; then - mapfile -t seed_refs < <( - git -C "${mirror_dir}" for-each-ref --format='%(refname)' refs/heads/ refs/tags/ - ) - echo "Seeding ${#seed_refs[@]} branches and tags in separate pushes..." - for seed_ref in "${seed_refs[@]}"; do - echo "Seeding ${seed_ref}..." - if ! git \ - -C "${mirror_dir}" \ - -c core.sshCommand="${gitlab_ssh_command}" \ - push --progress "${target_clone_url}" "+${seed_ref}:${seed_ref}"; then - echo "::error title=Mirror failed: ${source_name}::Unable to seed ${seed_ref} on GitLab." >&2 - exit 1 - fi - done - fi - - echo "Pushing the mirror to ${target_clone_url}..." - if ! git \ - -C "${mirror_dir}" \ - -c core.sshCommand="${gitlab_ssh_command}" \ - push --mirror --progress "${target_clone_url}"; then - echo "::error title=Mirror failed: ${source_name}::Unable to push the GitLab mirror." >&2 - exit 1 - fi - fi - rm -rf "${mirror_dir}" - echo "Completed mirror for ${source_name}." - ) 2>&1 | tee "${repository_log}" - repository_status="${PIPESTATUS[0]}" - set -e - - rm -rf "${temp_root}/repository.git" - if ((repository_status == 0)); then - successful_repository_count="$((successful_repository_count + 1))" - else - error_line="$(grep -E '^::error(::| )' "${repository_log}" | tail -n 1 || true)" - if [[ -n "${error_line}" ]]; then - failure_reason="${error_line##*::}" - else - failure_reason="Repository mirroring failed; inspect its log group." - fi - failure_reason="${failure_reason//$'\r'/ }" - failure_reason="${failure_reason//$'\n'/ }" - failed_repository_names+=("${source_name}") - failed_repository_reasons+=("${failure_reason}") - echo "Recorded mirror failure for ${source_name}; continuing with the next repository." - fi - rm -f "${repository_log}" - done - - failed_repository_count="${#failed_repository_names[@]}" - { - echo "## GitHub to GitLab mirror summary" - echo - echo "- Successful: ${successful_repository_count}" - echo "- Failed: ${failed_repository_count}" - if ((failed_repository_count > 0)); then - echo - echo "| Repository | Failure |" - echo "| --- | --- |" - for ((failure_index = 0; failure_index < failed_repository_count; failure_index++)); do - summary_name="${failed_repository_names[$failure_index]//|/\\|}" - summary_reason="${failed_repository_reasons[$failure_index]//|/\\|}" - printf '| %s | %s |\n' "\`${summary_name}\`" "${summary_reason}" - done - fi - } >> "${GITHUB_STEP_SUMMARY}" - - if ((failed_repository_count > 0)); then - echo "::error::Failed to mirror ${failed_repository_count} of ${repository_count} repositories." \ - "See the step summary." - exit 1 - fi - - echo "Mirrored all ${repository_count} repositories to GitLab." + run: bash .github/scripts/mirror-gitlab.sh diff --git a/.github/workflows/__pinact.yml b/.github/workflows/__pinact.yml index 7d91038a..a42e40dd 100644 --- a/.github/workflows/__pinact.yml +++ b/.github/workflows/__pinact.yml @@ -32,14 +32,51 @@ on: type: string repo: description: | - Specific repository to run pinact on (only use the repo name, omit the owner). - If specified, runs only on this repo instead of all org repos. + Specific owner/repository, or an unambiguous repository name from organizations.json. + If specified, runs only on this repo instead of all registered orgs. required: false default: '' type: string jobs: + setup: + permissions: + contents: read + runs-on: ubuntu-latest + outputs: + matrix: ${{ steps.plan.outputs.result }} + config: ${{ steps.plan.outputs.config }} + steps: + - name: Checkout controller + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + - name: Plan organizations or exact repository + id: plan + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + env: + REPOSITORY: ${{ inputs.repo }} + with: + github-token: ${{ secrets.GH_BOT_TOKEN }} + script: | + const {readOrganizations, discoverRepositories, resolveRepository, pinactConfiguration} = await import( + `${process.env.GITHUB_WORKSPACE}/.github/scripts/organization-repositories.mjs` + ); + const organizations = readOrganizations(); + core.setOutput('config', pinactConfiguration(organizations)); + if (!process.env.REPOSITORY.trim()) { + return {include: organizations.map(owner => ({owner, repository: ''}))}; + } + const repos = await discoverRepositories(github, organizations); + const repo = resolveRepository(repos, process.env.REPOSITORY); + return {include: [{owner: repo.owner.login, repository: repo.full_name}]}; + pinact: + name: Pinact (${{ matrix.repository || matrix.owner }}) + needs: setup + strategy: + fail-fast: false + max-parallel: 1 + matrix: ${{ fromJson(needs.setup.outputs.matrix) }} permissions: {} runs-on: ubuntu-latest steps: @@ -49,18 +86,10 @@ jobs: dryRun: ${{ github.event.inputs.dryRun }} gitAuthorEmail: ${{ secrets.GH_BOT_EMAIL }} gitAuthorName: ${{ vars.GH_BOT_NAME }} - githubOrg: ${{ github.repository_owner }} + githubOrg: ${{ matrix.owner }} includeForks: ${{ github.event.inputs.includeForks }} - pinactConfig: | - --- - version: 3 - ignore_actions: - - name: ${{ github.repository_owner }}/.* - ref: master - - name: Homebrew/actions/.* - ref: main - separator: " # " + pinactConfig: ${{ needs.setup.outputs.config }} pinactRepo: ${{ github.event.inputs.pinactRepo }} pinactVersion: ${{ github.event.inputs.pinactVersion }} - repo: ${{ github.event.inputs.repo && format('LizardByte/{0}', github.event.inputs.repo) || '' }} + repo: ${{ matrix.repository }} token: ${{ secrets.GH_BOT_TOKEN }} diff --git a/.github/workflows/__update-build-deps.yml b/.github/workflows/__update-build-deps.yml index 45340f08..979c27b7 100644 --- a/.github/workflows/__update-build-deps.yml +++ b/.github/workflows/__update-build-deps.yml @@ -19,13 +19,32 @@ on: jobs: update-build-deps: name: Update build-deps - ${{ inputs.repo }} - permissions: {} + permissions: + contents: read runs-on: ubuntu-latest steps: + - name: Checkout controller + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + - name: Resolve repository + id: repository + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + env: + REPOSITORY: ${{ inputs.repo }} + with: + github-token: ${{ secrets.GH_BOT_TOKEN }} + script: | + const {readOrganizations, discoverRepositories, resolveRepository} = await import( + `${process.env.GITHUB_WORKSPACE}/.github/scripts/organization-repositories.mjs` + ); + const repos = await discoverRepositories(github, readOrganizations()); + const repo = resolveRepository(repos, process.env.REPOSITORY); + core.setOutput('full-name', repo.full_name); + - name: Checkout uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: - repository: ${{ github.repository_owner }}/${{ inputs.repo }} + repository: ${{ steps.repository.outputs.full-name }} token: ${{ secrets.GH_BOT_TOKEN }} fetch-depth: 0 filter: blob:none @@ -72,7 +91,7 @@ jobs: github-token: ${{ secrets.GH_BOT_TOKEN }} script: | const fs = require('fs'); - const options = {owner: context.repo.owner, repo: 'build-deps'}; + const options = {owner: 'LizardByte', repo: 'build-deps'}; const tag = process.env.BUILD_DEPS_VERSION; if (!tag) { throw new Error('No build-deps submodule release tag was selected.'); diff --git a/.github/workflows/__update-cpm-cmake.yml b/.github/workflows/__update-cpm-cmake.yml index e11dd863..7628b510 100644 --- a/.github/workflows/__update-cpm-cmake.yml +++ b/.github/workflows/__update-cpm-cmake.yml @@ -18,35 +18,27 @@ jobs: outputs: matrix: ${{ steps.set-matrix.outputs.result }} has-repositories: ${{ steps.set-matrix.outputs.has-repositories }} - permissions: {} + permissions: + contents: read steps: + - name: Checkout controller + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - name: Get repos id: set-matrix uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 with: + github-token: ${{ secrets.GH_BOT_TOKEN }} script: | - const opts = github.rest.repos.listForOrg.endpoint.merge({ org: context.repo.owner }); - const repos = await github.paginate(opts); - - const matrix = { "include": [] }; - for (const repo of repos) { - const { data: values } = await github.request('GET /repos/{owner}/{repo}/properties/values', { - owner: context.repo.owner, - repo: repo.name - }); - const profiles = values.find(property => property.property_name === 'workflow_profiles')?.value ?? []; - if (!Array.isArray(profiles) || !profiles.every(profile => typeof profile === 'string')) { - throw new Error(`Invalid workflow_profiles custom property for ${repo.full_name}`); - } - if (profiles.includes('cpm')) { - matrix.include.push({ "repo": repo.name }); - } - } + const {readOrganizations, maintenanceMatrix} = await import( + `${process.env.GITHUB_WORKSPACE}/.github/scripts/organization-repositories.mjs` + ); + const matrix = await maintenanceMatrix(github, readOrganizations(), {profile: 'cpm'}); core.setOutput('has-repositories', matrix.include.length > 0); return matrix; test-matrix: - name: Test Matrix - ${{ matrix.repo }} + name: Test Matrix - ${{ matrix.full_name }} if: github.event_name == 'workflow_dispatch' && needs.setup-matrix.outputs.has-repositories == 'true' needs: setup-matrix permissions: {} @@ -59,7 +51,7 @@ jobs: run: echo ${{ matrix.repo }} update-cpm-cmake: - name: Update CPM.cmake - ${{ matrix.repo }} + name: Update CPM.cmake - ${{ matrix.full_name }} if: github.event_name == 'schedule' && needs.setup-matrix.outputs.has-repositories == 'true' needs: setup-matrix permissions: {} @@ -72,7 +64,7 @@ jobs: - name: Checkout uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: - repository: ${{ github.repository_owner }}/${{ matrix.repo }} + repository: ${{ matrix.full_name }} token: ${{ secrets.GH_BOT_TOKEN }} path: repo diff --git a/.github/workflows/__update_license_years.yml b/.github/workflows/__update_license_years.yml index 63cd8f80..bd05882a 100644 --- a/.github/workflows/__update_license_years.yml +++ b/.github/workflows/__update_license_years.yml @@ -15,28 +15,29 @@ jobs: runs-on: ubuntu-latest outputs: matrix: ${{ steps.set-matrix.outputs.result }} - permissions: {} + has-repositories: ${{ steps.set-matrix.outputs.has-repositories }} + permissions: + contents: read steps: - # get all repos in this org + - name: Checkout controller + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - name: Get repos id: set-matrix uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 with: + github-token: ${{ secrets.GH_BOT_TOKEN }} script: | - const opts = github.rest.repos.listForOrg.endpoint.merge({ org: context.repo.owner }); - const repos = await github.paginate(opts) - - // create a GitHub strategy matrix - let matrix = { "include": [] }; - for (const repo of repos) { - if (!repo.archived && !repo.fork) { - matrix.include.push({ "repo": repo.name }); - } - } - return matrix + const {readOrganizations, maintenanceMatrix} = await import( + `${process.env.GITHUB_WORKSPACE}/.github/scripts/organization-repositories.mjs` + ); + const matrix = await maintenanceMatrix(github, readOrganizations(), {includeForks: false}); + core.setOutput('has-repositories', matrix.include.length > 0); + return matrix; update: - name: Update License Year (${{ matrix.repo }}) + if: needs.setup-matrix.outputs.has-repositories == 'true' + name: Update License Year (${{ matrix.full_name }}) needs: setup-matrix permissions: {} runs-on: ubuntu-latest @@ -48,7 +49,8 @@ jobs: - name: Checkout uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: - repository: ${{ github.repository_owner }}/${{ matrix.repo }} + repository: ${{ matrix.full_name }} + token: ${{ secrets.GH_BOT_TOKEN }} fetch-depth: 0 filter: blob:none diff --git a/tests/gitlab-mirror.test.mjs b/tests/gitlab-mirror.test.mjs new file mode 100644 index 00000000..09746a3b --- /dev/null +++ b/tests/gitlab-mirror.test.mjs @@ -0,0 +1,167 @@ +import assert from 'node:assert/strict'; +import {readFileSync} from 'node:fs'; +import test from 'node:test'; + +import {GitLabMirrors, mirrorRepositories, mirrorTarget, prepareMirrors} from '../.github/scripts/gitlab-mirror.mjs'; + +const root = {id: 1, full_path: 'lizardbyte', visibility: 'public', parent_id: null}; +const group = {id: 2, full_path: 'lizardbyte/sunshinestream', visibility: 'public', parent_id: 1}; +function target(owner = 'SunshineStream', name = 'Sunshine', id = 99, visibility = 'public') { + return mirrorTarget({id, owner: {login: owner}, name, full_name: `${owner}/${name}`, + clone_url: `https://github.com/${owner}/${name}.git`, size: 100, visibility}, 'lizardbyte', 'LizardByte'); +} +function harness(routes) { + const calls = []; + const client = new GitLabMirrors({gitlabToken: 'gitlab-secret', githubToken: 'github-secret', sleep: async () => {}, + fetchImpl: async (url, options = {}) => { + const key = `${options.method ?? 'GET'} ${url.replace('https://gitlab.com/api/v4', '')}`; + calls.push({key, options, body: options.body ? JSON.parse(options.body) : undefined}); + assert.ok(key in routes, `Unexpected request: ${key}`); + const result = typeof routes[key] === 'function' ? routes[key](calls.at(-1)) : routes[key]; + if (result instanceof Error) throw result; + const [status, data] = result; + return {status, ok: status >= 200 && status < 300, json: async () => data}; + }}); + return {client, calls}; +} +const base = {'GET /groups/lizardbyte': [200, root], 'GET /groups/lizardbyte%2Fsunshinestream': [200, group]}; +const missing = {'GET /projects/lizardbyte%2Fsunshinestream%2FSunshine': [404, {}], + 'GET /projects/lizardbyte%2FSunshine': [404, {}]}; + +test('qualified exclusions do not hide another org sharing that repository name', () => { + const sources = ['LizardByte/ignored', 'Other/ignored', 'Other/repo-ghsa-2345-6789-cfgh'].map((name, index) => ({ + id: index + 1, owner: {login: name.split('/')[0]}, name: name.split('/')[1], full_name: name, + clone_url: `https://github.com/${name}.git`, size: 1, visibility: 'public', + })); + assert.deepEqual(mirrorRepositories(sources, ['lizardbyte/ignored'], 'lizardbyte', 'LizardByte').map(r => r.name), ['Other/ignored']); + assert.throws(() => mirrorRepositories(sources, ['ignored'], 'lizardbyte', 'LizardByte'), /qualified owner/); +}); + +test('target paths retain legacy spelling and stable normalization, with separate namespaces for same-name repos', () => { + assert.equal(target('LizardByte').targetGroup, 'lizardbyte'); + assert.equal(target().targetGroup, 'lizardbyte/sunshinestream'); + assert.equal(target('Themerr', '.github', 7).targetPath, 'dot-github-7'); + assert.equal(target('Other', '.github', 8).targetPath, 'dot-github-8'); + assert.equal(target('Other', 'repo.git').targetPath, 'repo.git-repo-99'); + assert.equal(target('Other', 'secret', 9, 'internal').targetVisibility, 'private'); + assert.throws(() => mirrorTarget({clone_url: 'bad'}, 'lizardbyte', 'LizardByte'), /Invalid mirror source/); +}); + +test('missing subgroups are created under the existing root and cached, retaining display names and visibility', async () => { + const {client, calls} = harness({...base, ...missing, + 'GET /groups/lizardbyte%2Fsunshinestream': [404, {}], 'POST /groups': [201, group]}); + assert.equal((await client.prepare(target())).namespaceId, 2); + assert.equal((await client.prepare(target())).namespaceId, 2); + const requests = calls.filter(c => c.key === 'POST /groups'); + assert.equal(requests.length, 1); + assert.deepEqual(requests[0].body, {parent_id: 1, name: 'SunshineStream', path: 'sunshinestream', visibility: 'public', auto_devops_enabled: false}); +}); + +test('group creation races are reread and inaccessible or incorrect parents fail closed', async () => { + let reads = 0; + const {client} = harness({...base, + 'GET /groups/lizardbyte%2Fsunshinestream': () => ++reads === 1 ? [404, {}] : [200, group], + 'POST /groups': [409, {}]}); + assert.equal((await client.group(group.full_path)).id, 2); + for (const response of [[403, {}], [200, {...group, parent_id: 9}], [200, {...group, full_path: 'wrong'}]]) { + const run = harness({...base, 'GET /groups/lizardbyte%2Fsunshinestream': response}); + await assert.rejects(run.client.prepare(target()), /GitLab group|subgroup parent/); + } +}); + +test('a redirected legacy GitHub URL proves identity before transferring the original GitLab project ID', async () => { + const legacy = {id: 42, path_with_namespace: 'lizardbyte/Sunshine', namespace: {id: 1}, + description: 'Mirror of https://github.com/LizardByte/Sunshine.git'}; + let polls = 0; + const {client, calls} = harness({...base, ...missing, + 'GET /projects/lizardbyte%2FSunshine': [200, legacy], + 'GET https://api.github.com/repos/LizardByte/Sunshine': [200, {id: 99}], + 'PUT /projects/42/transfer': [201, legacy], + 'GET /projects/42': () => [200, ++polls === 1 ? legacy : {...legacy, namespace: {id: 2}, path_with_namespace: 'lizardbyte/sunshinestream/Sunshine'}], + }); + assert.equal((await client.prepare(target())).namespaceId, 2); + assert.deepEqual(calls.find(c => c.key === 'PUT /projects/42/transfer').body, {namespace: 2}); + assert.ok(calls.every(c => !c.key.startsWith('DELETE') && c.key !== 'POST /projects')); +}); + +test('a same-name legacy project with a different GitHub ID is preserved without blocking a new subgroup mirror', async () => { + const {client, calls} = harness({...base, ...missing, 'GET /projects/lizardbyte%2FSunshine': [200, { + id: 42, path_with_namespace: 'lizardbyte/Sunshine', namespace: {id: 1}, description: 'GitHub repository ID: 123', + }]}); + assert.equal((await client.prepare(target())).namespaceId, 2); + assert.ok(calls.every(c => !c.key.startsWith('PUT'))); +}); + +test('private legacy mirrors are secured and verified before any namespace transfer', async () => { + const legacy = {id: 42, path_with_namespace: 'lizardbyte/Sunshine', namespace: {id: 1}, description: 'GitHub repository ID: 99'}; + const {client, calls} = harness({...base, ...missing, + 'GET /projects/lizardbyte%2FSunshine': [200, legacy], 'PUT /projects/42': [200, {...legacy, visibility: 'public'}]}); + await assert.rejects(client.prepare(target('SunshineStream', 'Sunshine', 99, 'private')), /privacy verification/); + assert.ok(!calls.some(c => c.key.endsWith('/transfer'))); +}); + +test('successful private migration verifies privacy before transfer and keeps the same GitLab ID', async () => { + const legacy = {id: 42, path_with_namespace: 'lizardbyte/Sunshine', namespace: {id: 1}, description: 'GitHub repository ID: 99'}; + const {client, calls} = harness({...base, ...missing, + 'GET /projects/lizardbyte%2FSunshine': [200, legacy], + 'PUT /projects/42': [200, {...legacy, visibility: 'private'}], + 'PUT /projects/42/transfer': [202, legacy], + 'GET /projects/42': [200, {...legacy, namespace: {id: 2}, path_with_namespace: 'lizardbyte/sunshinestream/Sunshine'}], + }); + assert.equal((await client.prepare(target('SunshineStream', 'Sunshine', 99, 'private'))).projectId, 42); + assert.ok(calls.findIndex(c => c.key === 'PUT /projects/42') < calls.findIndex(c => c.key.endsWith('/transfer'))); +}); + +test('legacy identity API failures and subgroup creation denial leave existing projects untouched', async () => { + const legacy = {id: 42, path_with_namespace: 'lizardbyte/Sunshine', namespace: {id: 1}, + description: 'Mirror of https://github.com/LizardByte/Sunshine.git'}; + const run = harness({...base, ...missing, 'GET /projects/lizardbyte%2FSunshine': [200, legacy], + 'GET https://api.github.com/repos/LizardByte/Sunshine': [403, {}]}); + await assert.rejects(run.client.prepare(target()), /Cannot verify the existing mirror/); + assert.ok(run.calls.every(c => c.key.startsWith('GET'))); + const denied = harness({...base, 'GET /groups/lizardbyte%2Fsunshinestream': [404, {}], 'POST /groups': [403, {}]}); + await assert.rejects(denied.client.prepare(target()), /Cannot create GitLab subgroup/); +}); + +test('unverifiable or conflicting destination identities, archived mirrors, and lookup failures are never overwritten', async () => { + const project = {id: 42, path_with_namespace: 'lizardbyte/sunshinestream/Sunshine', namespace: {id: 2}}; + for (const fields of [{description: 'manual project'}, {description: 'GitHub repository ID: 123'}, + {description: 'GitHub repository ID: 99', archived: true}]) { + const {client, calls} = harness({...base, 'GET /projects/lizardbyte%2Fsunshinestream%2FSunshine': [200, {...project, ...fields}]}); + await assert.rejects(client.prepare(target()), /identity|different GitHub repository|archived/); + assert.ok(calls.every(c => c.key.startsWith('GET'))); + } + const {client} = harness({...base, ...missing, 'GET /projects/lizardbyte%2FSunshine': [403, {}]}); + await assert.rejects(client.prepare(target()), /Cannot verify legacy/); +}); + +test('transfer errors and incomplete asynchronous transfers fail instead of creating replacement projects', async () => { + const legacy = {id: 42, path_with_namespace: 'lizardbyte/Sunshine', namespace: {id: 1}, description: 'GitHub repository ID: 99'}; + for (const status of [403, 202]) { + const {client} = harness({...base, ...missing, + 'GET /projects/lizardbyte%2FSunshine': [200, legacy], + 'PUT /projects/42/transfer': [status, {}], 'GET /projects/42': [200, legacy]}); + await assert.rejects(client.prepare(target()), /transfer failed|did not complete/); + } +}); + +test('per-repository preparation failures are saved while other mirrors continue', async () => { + const run = harness({...base, ...missing, 'GET /projects/lizardbyte%2FSunshine': [403, {}]}); + const results = await prepareMirrors([target(), target('LizardByte', 'New', 100)], { + gitlabToken: 'secret', githubToken: 'secret', fetchImpl: async (url, options) => { + if (url.endsWith('lizardbyte%2FNew')) return {status: 404, json: async () => ({})}; + return run.client.fetch(url, options); + }, + }); + assert.match(results[0].preparationError, /Cannot verify legacy/); + assert.equal(results[1].namespaceId, 1); +}); + +test('Git push remains behind preparation and private visibility verification, with large-import support intact', () => { + const script = readFileSync('.github/scripts/mirror-gitlab.sh', 'utf8'); + assert.ok(script.indexOf('preparation_error}') < script.indexOf('Checking for the GitLab project')); + assert.ok(script.indexOf('GitLab privacy verification failed') < script.indexOf('clone --mirror')); + assert.match(script, /namespace_id/); + assert.match(script, /push --mirror/); + assert.match(script, /import\/git/); +}); diff --git a/tests/organization-repositories.test.mjs b/tests/organization-repositories.test.mjs new file mode 100644 index 00000000..67a33a39 --- /dev/null +++ b/tests/organization-repositories.test.mjs @@ -0,0 +1,108 @@ +import assert from 'node:assert/strict'; +import {mkdtempSync, readFileSync, rmSync, writeFileSync} from 'node:fs'; +import {tmpdir} from 'node:os'; +import {join} from 'node:path'; +import test from 'node:test'; + +import {discoverRepositories, maintenanceMatrix, pinactConfiguration, readOrganizations, + resolveRepository} from '../.github/scripts/organization-repositories.mjs'; + +const organizations = ['Source', 'Project']; +function repo(owner, name, id, extra = {}) { + return {id, owner: {login: owner}, name, full_name: `${owner}/${name}`, visibility: 'public', ...extra}; +} +function client(repos, values = {}) { + const calls = []; + return {calls, github: { + rest: {repos: {listForOrg: {endpoint: {merge: options => options}}}}, + paginate: async options => { calls.push(options); return repos.filter(r => r.owner.login === options.org); }, + request: async (route, coordinates) => { + assert.equal(route, 'GET /repos/{owner}/{repo}/properties/values'); + const value = values[`${coordinates.owner}/${coordinates.repo}`]; + if (value instanceof Error) throw value; + return {data: value === undefined ? [] : [{property_name: 'workflow_profiles', value}]}; + }, + }}; +} + +test('registry rejects malformed and duplicate names without duplicating its organization list', () => { + const directory = mkdtempSync(join(tmpdir(), 'organization-registry-')); + const path = join(directory, 'organizations.json'); + try { + for (const value of [[], {}, null, ['bad/org'], [7], ['Source', 'source']]) { + writeFileSync(path, JSON.stringify(value)); + assert.throws(() => readOrganizations(path), /Invalid organization registry/); + } + writeFileSync(path, JSON.stringify(organizations)); + assert.deepEqual(readOrganizations(path), organizations); + } finally { rmSync(directory, {recursive: true, force: true}); } +}); + +test('cross-org discovery retains same-name projects and filters private/archived repos from public matrices', async () => { + const repos = [repo('Source', 'shared', 1), repo('Project', 'shared', 2), + repo('Source', 'private', 3, {visibility: 'private'}), repo('Project', 'archived', 4, {archived: true}), + repo('Project', 'fork', 5, {fork: true})]; + const {github, calls} = client(repos); + assert.deepEqual((await discoverRepositories(github, organizations)).map(r => r.id), [1, 2, 5]); + assert.deepEqual(calls, organizations.map(org => ({org, per_page: 100}))); + assert.equal((await discoverRepositories(github, organizations, {includePrivate: true, includeArchived: true})).length, 5); + assert.deepEqual((await discoverRepositories(github, organizations, {includeForks: false})).map(r => r.id), [1, 2]); +}); + +test('discovery fails on API errors, invalid identity, or conflicting repository IDs', async () => { + const {github} = client([repo('Source', 'a', 1), repo('Project', 'b', 1)]); + await assert.rejects(discoverRepositories(github, organizations), /ownership changed/); + const bad = client([repo('Source', 'a', 1, {full_name: 'Other/a'})]); + await assert.rejects(discoverRepositories(bad.github, organizations), /Invalid repository identity/); + github.paginate = async () => { throw new Error('Access denied'); }; + await assert.rejects(discoverRepositories(github, organizations), /Access denied/); +}); + +test('CPM selection reads custom properties from each actual owner, independently of topics and sync_mode', async () => { + const {github} = client([repo('Source', 'a', 1), repo('Project', 'b', 2, {topics: ['cpm']})], { + 'Source/a': ['cpm', 'cpp_style'], 'Project/b': ['cpp_style'], + }); + assert.deepEqual(await maintenanceMatrix(github, organizations, {profile: 'cpm'}), { + include: [{owner: 'Source', repo: 'a', full_name: 'Source/a'}], + }); + assert.deepEqual(await maintenanceMatrix(github, organizations, {profile: 'unset'}), {include: []}); +}); + +test('CPM rejects malformed properties and inaccessible property reads', async () => { + for (const value of ['cpm', [null], new Error('Property access denied')]) { + const {github} = client([repo('Project', 'a', 1)], {'Project/a': value}); + await assert.rejects(maintenanceMatrix(github, organizations, {profile: 'cpm'}), + /Invalid workflow_profiles|Property access denied/); + } +}); + +test('license and stale matrices retain fork and issue eligibility rules and handle empty organizations', async () => { + const {github} = client([repo('Source', 'fork', 1, {fork: true}), repo('Project', 'no-issues', 2, {has_issues: false})]); + assert.equal((await maintenanceMatrix(github, organizations, {includeForks: false})).include[0].owner, 'Project'); + assert.equal((await maintenanceMatrix(github, organizations, {issuesOnly: true})).include[0].repo, 'fork'); + assert.deepEqual(await maintenanceMatrix(client([]).github, organizations), {include: []}); +}); + +test('short-name entry points follow transfers and reject ambiguous or out-of-registry targets', () => { + const repos = [repo('Source', 'same', 1), repo('Project', 'same', 2), repo('Project', 'Sunshine', 3)]; + assert.equal(resolveRepository(repos, 'sunshine').full_name, 'Project/Sunshine'); + assert.equal(resolveRepository(repos, 'PROJECT/same').id, 2); + for (const value of ['same', 'Other/Sunshine', '', '../Sunshine']) { + assert.throws(() => resolveRepository(repos, value), /exactly one eligible repository/); + } + assert.equal(JSON.parse(pinactConfiguration([...organizations, 'FutureOrg'])).ignore_actions[2].name, 'FutureOrg/.*'); +}); + +test('controller workflows wire qualified targets, bot credentials, and safe empty matrices', () => { + for (const file of ['__issues-stale.yml', '__update-cpm-cmake.yml', '__update_license_years.yml']) { + const source = readFileSync(`.github/workflows/${file}`, 'utf8'); + assert.match(source, /maintenanceMatrix\(github, readOrganizations\(\)/); + assert.match(source, /github-token: \$\{\{ secrets.GH_BOT_TOKEN \}\}/); + assert.match(source, /has-repositories == 'true'/); + assert.doesNotMatch(source, /repository: \$\{\{ github.repository_owner \}\}/); + } + assert.match(readFileSync('.github/workflows/__issues-stale.yml', 'utf8'), /repo-owner: \$\{\{ matrix.owner \}\}/); + const buildDeps = readFileSync('.github/workflows/__update-build-deps.yml', 'utf8'); + assert.match(buildDeps, /resolveRepository\(repos, process.env.REPOSITORY\)/); + assert.match(buildDeps, /const options = \{owner: 'LizardByte', repo: 'build-deps'\}/); +}); diff --git a/tests/workflow-profile-selection.test.mjs b/tests/workflow-profile-selection.test.mjs index 424bf00f..5d460011 100644 --- a/tests/workflow-profile-selection.test.mjs +++ b/tests/workflow-profile-selection.test.mjs @@ -76,49 +76,10 @@ for (const [file, profile] of profiles) { }); } -const cpm = loadWorkflow('__update-cpm-cmake.yml'); -async function cpmMatrix(repositories, valuesByName, error) { - const outputs = {}; - const github = { - rest: {repos: {listForOrg: {endpoint: {merge: options => options}}}}, - paginate: async options => { - assert.deepEqual(options, {org: 'source-org'}); - return repositories; - }, - request: async (route, {owner, repo}) => { - assert.equal(route, propertyRoute); - assert.equal(owner, 'source-org'); - if (error) throw error; - return {data: profileValues(valuesByName[repo])}; - }, - }; - const matrix = await cpm.execute(github, {repo: {owner: 'source-org'}}, { - setOutput: (name, value) => { outputs[name] = value; }, - }); - return {matrix, outputs}; -} - -test('CPM maintenance selects custom properties instead of public topics', async () => { - const repos = [{name: 'selected', topics: []}, {name: 'topic-only', topics: ['cpm']}, {name: 'unset'}]; - assert.deepEqual(await cpmMatrix(repos, {selected: ['cpm', 'cpp_style'], 'topic-only': ['cpp_style']}), { - matrix: {include: [{repo: 'selected'}]}, outputs: {'has-repositories': true}, - }); -}); - -test('CPM maintenance reports empty selections so both matrix jobs can skip safely', async () => { - assert.deepEqual(await cpmMatrix([{name: 'unset'}, {name: 'null'}], {null: null}), { - matrix: {include: []}, outputs: {'has-repositories': false}, - }); - assert.deepEqual(await cpmMatrix([], {}), { - matrix: {include: []}, outputs: {'has-repositories': false}, - }); - assert.equal(cpm.source.match(/if:.*needs.setup-matrix.outputs.has-repositories == 'true'/g).length, 2); -}); - -test('CPM maintenance fails for malformed selections and inaccessible properties', async () => { - await assert.rejects(cpmMatrix([{name: 'bad', full_name: 'source-org/bad'}], {bad: 'cpm'}), - /Invalid workflow_profiles custom property for source-org\/bad/); - await assert.rejects(cpmMatrix([{name: 'bad'}], {}, new Error('Access denied')), /Access denied/); +test('CPM maintenance delegates profile and organization selection to the shared discovery helper', () => { + const source = readFileSync('.github/workflows/__update-cpm-cmake.yml', 'utf8'); + assert.match(source, /maintenanceMatrix\(github, readOrganizations\(\), \{profile: 'cpm'\}\)/); + assert.equal(source.match(/if:.*needs.setup-matrix.outputs.has-repositories == 'true'/g).length, 2); }); test('workflow automation contains no repository-topic API calls or legacy profile guidance', () => { From fd640ae071d971c842144f3581fbf8ccb692d522 Mon Sep 17 00:00:00 2001 From: ReenigneArcher <42013603+ReenigneArcher@users.noreply.github.com> Date: Sun, 8 Nov 2026 17:47:02 -0500 Subject: [PATCH 2/4] fix: address mirror preparation analysis findings --- .github/scripts/gitlab-mirror.mjs | 20 ++++++++++---------- .github/scripts/mirror-gitlab.sh | 2 +- tests/gitlab-mirror.test.mjs | 8 ++++++++ 3 files changed, 19 insertions(+), 11 deletions(-) diff --git a/.github/scripts/gitlab-mirror.mjs b/.github/scripts/gitlab-mirror.mjs index 880038a4..2bc35d33 100644 --- a/.github/scripts/gitlab-mirror.mjs +++ b/.github/scripts/gitlab-mirror.mjs @@ -23,7 +23,7 @@ export function mirrorTarget(repo, rootGroup, sourceOrganization) { const prefixNames = {'.': 'dot-', '-': 'dash-', '_': 'underscore-'}; const lowerName = repo.name.toLowerCase(); let path = lowerName.replace(/^[._-]+/, prefix => [...prefix].map(char => prefixNames[char]).join('')) - .replace(/[^a-z0-9_.-]+/g, '-').replace(/[._-]{2,}/g, '-').replace(/[._-]+$/g, ''); + .replace(/[^a-z0-9_.-]+/g, '-').replace(/[._-]{2,}/g, '-').replace(/[._-]$/, ''); if (path.endsWith('.git') || path.endsWith('.atom')) path += '-repo'; if (!path) path = 'repository'; const transformed = path !== lowerName; @@ -176,14 +176,13 @@ export class GitLabMirrors { } /** GitLab transfers can be asynchronous; verify the same project reached the requested namespace. */ - async waitForTransfer(projectId, namespaceId, path) { - for (let attempt = 0; attempt < 30; attempt++) { - const result = await this.request('GET', `/projects/${projectId}`); - if (result.status !== 200 || result.data.id !== projectId) throw new Error('GitLab transfer verification failed'); - if (result.data.namespace?.id === namespaceId && result.data.path_with_namespace === path) return; - await this.sleep(2000); - } - throw new Error('GitLab mirror transfer did not complete within one minute'); + async waitForTransfer(projectId, namespaceId, path, attemptsRemaining = 30) { + const result = await this.request('GET', `/projects/${projectId}`); + if (result.status !== 200 || result.data.id !== projectId) throw new Error('GitLab transfer verification failed'); + if (result.data.namespace?.id === namespaceId && result.data.path_with_namespace === path) return; + if (attemptsRemaining <= 1) throw new Error('GitLab mirror transfer did not complete within one minute'); + await this.sleep(2000); + return this.waitForTransfer(projectId, namespaceId, path, attemptsRemaining - 1); } } @@ -200,7 +199,8 @@ export async function prepareMirrors(repositories, options) { } if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { - const path = process.argv[2]; + if (process.argv.length !== 2) throw new Error('Mirror preparation does not accept CLI arguments'); + const path = new URL('../../repositories.json', import.meta.url); const prepared = await prepareMirrors(JSON.parse(readFileSync(path, 'utf8')), { gitlabToken: process.env.GITLAB_TOKEN, githubToken: process.env.GITHUB_TOKEN, rootGroup: process.env.GITLAB_GROUP, }); diff --git a/.github/scripts/mirror-gitlab.sh b/.github/scripts/mirror-gitlab.sh index 1e58679d..10333d19 100644 --- a/.github/scripts/mirror-gitlab.sh +++ b/.github/scripts/mirror-gitlab.sh @@ -129,7 +129,7 @@ wait_for_gitlab_import() { return 1 } -node .github/scripts/gitlab-mirror.mjs "${repository_file}" +node .github/scripts/gitlab-mirror.mjs github_auth="$(printf 'x-access-token:%s' "${GITHUB_TOKEN}" | base64 --wrap=0)" echo "::add-mask::${github_auth}" diff --git a/tests/gitlab-mirror.test.mjs b/tests/gitlab-mirror.test.mjs index 09746a3b..8ec3538c 100644 --- a/tests/gitlab-mirror.test.mjs +++ b/tests/gitlab-mirror.test.mjs @@ -1,4 +1,5 @@ import assert from 'node:assert/strict'; +import {spawnSync} from 'node:child_process'; import {readFileSync} from 'node:fs'; import test from 'node:test'; @@ -43,10 +44,17 @@ test('target paths retain legacy spelling and stable normalization, with separat assert.equal(target('Themerr', '.github', 7).targetPath, 'dot-github-7'); assert.equal(target('Other', '.github', 8).targetPath, 'dot-github-8'); assert.equal(target('Other', 'repo.git').targetPath, 'repo.git-repo-99'); + assert.equal(target('Other', 'repo._-').targetPath, 'repo-99'); assert.equal(target('Other', 'secret', 9, 'internal').targetVisibility, 'private'); assert.throws(() => mirrorTarget({clone_url: 'bad'}, 'lizardbyte', 'LizardByte'), /Invalid mirror source/); }); +test('mirror preparation rejects CLI paths before reading inventory or making requests', () => { + const result = spawnSync(process.execPath, ['.github/scripts/gitlab-mirror.mjs', '../unexpected.json'], {encoding: 'utf8'}); + assert.equal(result.status, 1); + assert.match(result.stderr, /does not accept CLI arguments/); +}); + test('missing subgroups are created under the existing root and cached, retaining display names and visibility', async () => { const {client, calls} = harness({...base, ...missing, 'GET /groups/lizardbyte%2Fsunshinestream': [404, {}], 'POST /groups': [201, group]}); From 4a63ff7951f8bba6901c84a4d3b9fbfeefd61b5b Mon Sep 17 00:00:00 2001 From: ReenigneArcher <42013603+ReenigneArcher@users.noreply.github.com> Date: Sun, 8 Nov 2026 18:57:24 -0500 Subject: [PATCH 3/4] ci: use explicit repository choices for build-deps --- .github/workflows/README.md | 6 ++--- .github/workflows/__update-build-deps.yml | 31 +++++------------------ tests/organization-repositories.test.mjs | 3 ++- 3 files changed, 11 insertions(+), 29 deletions(-) diff --git a/.github/workflows/README.md b/.github/workflows/README.md index baf0fad2..04dd8aa8 100644 --- a/.github/workflows/README.md +++ b/.github/workflows/README.md @@ -33,12 +33,12 @@ requires `GH_TOKEN` with `write:packages` access to the existing LizardByte pack passes `GH_BOT_TOKEN`. npmjs trusted-publisher bindings and external service installations must also be updated for the destination owner when transferring a repository. -Central stale-issue, license-year, CPM, Pinact, and build-deps workflows read `organizations.json` as well. +Central stale-issue, license-year, CPM, and Pinact workflows read `organizations.json` as well. Repository matrices use qualified identities so matching names in different orgs remain distinct. Public maintenance matrices exclude private and archived repositories; license updates also exclude forks. CPM continues to select the `cpm` custom-property profile. Manual stale/CPM dispatches preview their matrices. -The build-deps updater retains its existing project choices and resolves their current owners through the -registry; dependency releases still come from `LizardByte/build-deps`. Pinact accepts qualified repo names +The build-deps updater uses explicit `owner/repository` choices, which must be updated after a transfer; +dependency releases still come from `LizardByte/build-deps`. Pinact accepts qualified repo names or unambiguous short names and preserves shared `master` references across all registered orgs. The GitLab mirror reads all registered orgs, including private, archived, and forked source repositories. diff --git a/.github/workflows/__update-build-deps.yml b/.github/workflows/__update-build-deps.yml index 979c27b7..be5cd00a 100644 --- a/.github/workflows/__update-build-deps.yml +++ b/.github/workflows/__update-build-deps.yml @@ -8,43 +8,24 @@ on: workflow_dispatch: inputs: repo: - description: 'Repository to update.' + description: 'Repository to update (owner/repository).' required: true type: choice options: - - Sunshine - - Koko - - One + - LizardByte/Sunshine + - LizardByte/Koko + - LizardByte/One jobs: update-build-deps: name: Update build-deps - ${{ inputs.repo }} - permissions: - contents: read + permissions: {} runs-on: ubuntu-latest steps: - - name: Checkout controller - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - - name: Resolve repository - id: repository - uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 - env: - REPOSITORY: ${{ inputs.repo }} - with: - github-token: ${{ secrets.GH_BOT_TOKEN }} - script: | - const {readOrganizations, discoverRepositories, resolveRepository} = await import( - `${process.env.GITHUB_WORKSPACE}/.github/scripts/organization-repositories.mjs` - ); - const repos = await discoverRepositories(github, readOrganizations()); - const repo = resolveRepository(repos, process.env.REPOSITORY); - core.setOutput('full-name', repo.full_name); - - name: Checkout uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: - repository: ${{ steps.repository.outputs.full-name }} + repository: ${{ inputs.repo }} token: ${{ secrets.GH_BOT_TOKEN }} fetch-depth: 0 filter: blob:none diff --git a/tests/organization-repositories.test.mjs b/tests/organization-repositories.test.mjs index 67a33a39..b9948cd8 100644 --- a/tests/organization-repositories.test.mjs +++ b/tests/organization-repositories.test.mjs @@ -103,6 +103,7 @@ test('controller workflows wire qualified targets, bot credentials, and safe emp } assert.match(readFileSync('.github/workflows/__issues-stale.yml', 'utf8'), /repo-owner: \$\{\{ matrix.owner \}\}/); const buildDeps = readFileSync('.github/workflows/__update-build-deps.yml', 'utf8'); - assert.match(buildDeps, /resolveRepository\(repos, process.env.REPOSITORY\)/); + assert.match(buildDeps, /repository: \$\{\{ inputs.repo \}\}/); + assert.doesNotMatch(buildDeps, /discoverRepositories|resolveRepository|Checkout controller/); assert.match(buildDeps, /const options = \{owner: 'LizardByte', repo: 'build-deps'\}/); }); From 7a02e1141bf20f3f3e9858a0af87bcf83abd68c6 Mon Sep 17 00:00:00 2001 From: ReenigneArcher <42013603+ReenigneArcher@users.noreply.github.com> Date: Sun, 8 Nov 2026 18:57:24 -0500 Subject: [PATCH 4/4] ci: remove stale GitLab mirror exclusions --- .github/repositories-to-ignore.json | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.github/repositories-to-ignore.json b/.github/repositories-to-ignore.json index e16494e1..0992bbec 100644 --- a/.github/repositories-to-ignore.json +++ b/.github/repositories-to-ignore.json @@ -1,6 +1,3 @@ [ - "LizardByte/homebrew-core", - "LizardByte/winget-pkgs", - "LizardByte/Virtual-Gamepad-Emulation-Client", - "LizardByte/Virtual-Gamepad-Emulation-dotnet" + "LizardByte/Virtual-Gamepad-Emulation-Client" ]