From 6b4f6d976504060832f6f5be240f51877b671896 Mon Sep 17 00:00:00 2001 From: Ryan Krattiger Date: Wed, 15 May 2024 14:14:16 -0500 Subject: [PATCH] Cron job to start rebuild everything pipelines --- .github/workflows/custom_docker_builds.yml | 2 + images/git-remote-sync/Dockerfile | 9 ++ images/git-remote-sync/requirements.txt | 1 + images/git-remote-sync/sync-git-remotes.py | 116 ++++++++++++++++++ .../custom/rebuild-everything/cron-jobs.yaml | 42 +++++++ .../rebuild-everything/sealed-secrets.yaml | 14 +++ 6 files changed, 184 insertions(+) create mode 100644 images/git-remote-sync/Dockerfile create mode 100644 images/git-remote-sync/requirements.txt create mode 100644 images/git-remote-sync/sync-git-remotes.py create mode 100644 k8s/production/custom/rebuild-everything/cron-jobs.yaml create mode 100644 k8s/production/custom/rebuild-everything/sealed-secrets.yaml diff --git a/.github/workflows/custom_docker_builds.yml b/.github/workflows/custom_docker_builds.yml index 8733d3290..a46d22eb7 100644 --- a/.github/workflows/custom_docker_builds.yml +++ b/.github/workflows/custom_docker_builds.yml @@ -45,6 +45,8 @@ jobs: image-tags: ghcr.io/spack/django:0.2.8 - docker-image: ./images/ci-prune-buildcache image-tags: ghcr.io/spack/ci-prune-buildcache:0.0.3 + - docker-image: ./images/git-remote-sync + image-tags: ghcr.io/spack/git-remote-sync:0.0.1 steps: - name: Checkout uses: actions/checkout@1d96c772d19495a3b5c517cd2bc0cb401ea0529f # v4.1.3 diff --git a/images/git-remote-sync/Dockerfile b/images/git-remote-sync/Dockerfile new file mode 100644 index 000000000..78b981901 --- /dev/null +++ b/images/git-remote-sync/Dockerfile @@ -0,0 +1,9 @@ +FROM python:3.10 + +WORKDIR /scripts/ +COPY requirements.txt ./ +RUN pip install --no-cache-dir -r requirements.txt +COPY sync-git-remotes.py ./ + +ENTRYPOINT [ "python", "./sync-git-remotes.py"] + diff --git a/images/git-remote-sync/requirements.txt b/images/git-remote-sync/requirements.txt new file mode 100644 index 000000000..eb6f17ada --- /dev/null +++ b/images/git-remote-sync/requirements.txt @@ -0,0 +1 @@ +sentry-sdk==1.32.0 diff --git a/images/git-remote-sync/sync-git-remotes.py b/images/git-remote-sync/sync-git-remotes.py new file mode 100644 index 000000000..7546f12d4 --- /dev/null +++ b/images/git-remote-sync/sync-git-remotes.py @@ -0,0 +1,116 @@ +import argparse +import base64 +import os +import re +import subprocess +import sys +import tempfile +from contextlib import contextmanager + +try: + import sentry_sdk + sentry_sdk.init( + # This cron job only runs once weekly, + # so just record all transactions. + traces_sample_rate=1.0, + ) +except Exception: + print("Could not configure sentry.") + + +# Copied from SpackCIBridge.py +def setup_ssh(self, ssh_key_base64): + """Start the ssh agent.""" + print("Starting ssh-agent") + output = subprocess.run(["ssh-agent", "-s"], check=True, stdout=subprocess.PIPE).stdout + + # Search for PID in output. + pid_regexp = re.compile(r"SSH_AGENT_PID=([0-9]+)") + match = pid_regexp.search(output.decode("utf-8")) + if match is None: + print("WARNING: could not detect ssh-agent PID.", file=sys.stderr) + print("ssh-agent will not be killed upon program termination", file=sys.stderr) + else: + pid = match.group(1) + os.environ["SSH_AGENT_PID"] = pid + self.cleanup_ssh_agent = True + + # Search for socket in output. + socket_regexp = re.compile(r"SSH_AUTH_SOCK=([^;]+);") + match = socket_regexp.search(output.decode("utf-8")) + if match is None: + print("WARNING: could not detect ssh-agent socket.", file=sys.stderr) + print("Key will be added to caller's ssh-agent (if any)", file=sys.stderr) + else: + socket = match.group(1) + os.environ["SSH_AUTH_SOCK"] = socket + + # Add the key. + ssh_key = base64.b64decode(ssh_key_base64) + ssh_key = ssh_key.replace(b"\r", b"") + with tempfile.NamedTemporaryFile() as fp: + fp.write(ssh_key) + fp.seek(0) + subprocess.run(["ssh-add", fp.name], check=True) + + +def setup_repo(pull_remote, push_remote, branch): + subprocess.run(["git", "init"], check=True) + subprocess.run(["git", "config", "user.email", "noreply@spack.io"], check=True) + subprocess.run(["git", "config", "user.name", "spackbot"], check=True) + subprocess.run(["git", "config", "advice.detachedHead", "false"], check=True) + + subprocess.run(["git", "remote", "add", "origin", pull_remote], check=True) + subprocess.run(["git", "remote", "set-url", "--push", "origin", push_remote], check=True) + + fetch_args = ["git", "fetch", "-q", "origin", branch] + subprocess.run(fetch_args, check=True, stdout=subprocess.PIPE).stdout + subprocess.run(["git", "checkout", branch], check=True) + + +@contextmanager +def chworkingdir(path): + save_dir = os.path.realpath(os.curdir) + if path: + if not os.path.exists(path): + os.makedirs(path) + os.chdir(path) + yield path + else: + with tempfile.TemporaryDirectory() as tmpdir: + os.chdir(tmpdir) + yield tmpdir + + os.chdir(save_dir) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="Pull and repush a branch from one remote to another") + parser.add_argument("--push-remote", help="Host to push to") + parser.add_argument("--push-branch", help="Name of pushed branch") + + parser.add_argument("--tag", action="store_true", help="Use a tag instead of a branch alias") + + parser.add_argument("--pull-remote", help="Host to pull to") + parser.add_argument("--pull-branch", help="Name of pulled branch") + + parser.add_argument("--working-dir", help="Set the working directory") + + args = parser.parse_args() + + # Configure ssh for pushing to gitlab + ssh_key_base64 = os.getenv("GITLAB_SSH_KEY_BASE64") + if ssh_key_base64 is None: + raise Exception("GITLAB_SSH_KEY_BASE64 environment is not set") + setup_ssh(ssh_key_base64) + + with chworkingdir(args.working_dir): + setup_repo(args.pull_remote, args.push_remote, args.pull_branch) + + if args.tag: + subprocess.run(["git", "tag", args.push_branch], check=True) + else: + subprocess.run(["git", "checkout", "-b", args.push_branch], check=True) + + # Push the renamed branch to the push remote + subprocess.run(["git", "push", "-f", "origin", args.push_branch], check=True) diff --git a/k8s/production/custom/rebuild-everything/cron-jobs.yaml b/k8s/production/custom/rebuild-everything/cron-jobs.yaml new file mode 100644 index 000000000..3b8575ce3 --- /dev/null +++ b/k8s/production/custom/rebuild-everything/cron-jobs.yaml @@ -0,0 +1,42 @@ +--- +apiVersion: batch/v1 +kind: CronJob +metadata: + name: rebuild-everything + namespace: custom +spec: + schedule: "0 0 */14 * 6" + concurrencyPolicy: Forbid + jobTemplate: + spec: + activeDeadlineSeconds: 1200 # terminate any running job after 20 minutes + backoffLimit: 0 + template: + spec: + restartPolicy: Never + containers: + - name: sync + image: ghcr.io/spack/git-remote-sync:0.0.1 + imagePullPolicy: IfNotPresent + resources: + requests: + cpu: 500m + memory: 500M + env: + # This secret is double base64 encoded + - name: GITLAB_SSH_KEY_BASE64 + valueFrom: + secretKeyRef: + name: rebuild-everything + key: gitlab-ssh-key + args: + - "--push-mirror" + - "git@ssh.gitlab.spack.io:spack/spack.git" + - "--push-branch" + - "develop-re-$(date --iso-8601)" + - "--pull-mirror" + - "https://github.com/spack/spack.git" + - "--pull-branch" + - "develop" + nodeSelector: + spack.io/node-pool: base diff --git a/k8s/production/custom/rebuild-everything/sealed-secrets.yaml b/k8s/production/custom/rebuild-everything/sealed-secrets.yaml new file mode 100644 index 000000000..128e715d1 --- /dev/null +++ b/k8s/production/custom/rebuild-everything/sealed-secrets.yaml @@ -0,0 +1,14 @@ +apiVersion: bitnami.com/v1alpha1 +kind: SealedSecret +metadata: + name: rebuild-everything + namespace: custom +spec: + encryptedData: + # This token must have repo:status permissions + gitlab-ssh-key: AgBOg3sj4PYdQRvGEISHOUQVRa1FSEX+oiILWWrvGGUZ5jKrcCX+7vNKJZx/9MQIkMhZGFfkfvCyUjcu7ztehdfljnIZMdnkC3JvYp4l8AmGJhC6S3rJwHbQZaAiv+zWTrHHLztGI5rm9CnhzzLM5eAdWrFy8lPYoF89dzQuRrJgNQ/7eDpfEhtfXVQkNO/pzLTBTlQ4vBjj/eC3d34SuYygfmUXmmJvYMrw/3NsRFOBiUQiywU+m13VA2l+xRG9CNSBMxktyLj1oW7MKLFuIRb0Jnv5hOg6rBoXhRATaHeTj2n4VNr4ztbZtSeq2k/ZL1bgk7nppfSIc/K8crpBXApQBHkORjQnhnGGWSZqii5LC7sU0tRQOUu0987xAtO6V/EvpAa+ZsthhGyCv1E7VK/NemFVHipf+2cO39y7AC5Xk6wUn+FpW18yWPpPofylvIdLjmh5qDlqKPx8ZLgcwx+/1IHjmod7iSKHFw8wWQgX8W0NmAjqWmblHaCoXLrqEEOvQIyT0LH7+VjRJtprEnyPUFTAy75Kk6VFa3UkBkzorsdGw7R2vhEVVzV/WnZSY7rAL//CLdiXP/ut/tf8aHj4A1zezSRo3nm7lXl7DfCa9NAUNg+Y20AGpNKKAGmWRwAcxw92jmK09lrWVmFxAxRoxz3HexzpAvJxb2YEU9/C4gkP0+OYZ1pqKQI+UfISjrzNco3eYWXX9rrTEYqrcHu/FCRKTGp2oKndsAH5ehPONo8nV23uatUS7uhBfZCF0MfqZrfLaPhCMxiMlvuVtFc1WmtzJ4NwxoaBXhrtYgwVP+k2nPXEVYvbwNNG7/1UdGdgdJMhJmvkoPgwXbRF9PddI+9tm4jAp6fXj837yAeVA5Je/kNdXhlw4LmADm7J5E1jNIDMJg6MqGAOZ1aNJypb+YVZ/pbULTUnP963jlzzp1FFuHT/CnaB14/MeXWYCBtqUHiQ5pExxSxgaI4Fjm63loQSEaDiO3+PkghsNWkNQ10liFznAGSoABfiIaOcadnBWsI2zJ+bE1ST2U8mVF2ClvzloJ8lk23p3MvstuliEQqRlKLDWPERAEBIljd5GtQBH2LVq7FsKoHWIqLSXjdUxNE13ns+ngDwCIA5snoKyqxec9IJLpBaTMnyDStMdLbxwuE6PJRVYOoktTa+OV1Ja6yBYmyjAQG5hzJMxY1FBQZTSUlB5K029sT5s35o8aRrpiwatcQLKZuKqw6SI0wtSgKIkL5f3BnuwGeUfeUPveW0IAxHH0dUbQEZkNyqjgQbVq6Oui4LGp9ZLgFLpNYmVC0O62y1cOtJmmo5jztPJS+zJ0zJKbgArKUgsZa7+9BVJefTTmY43C1TttFbVQhg5+O0KY9xuoxfxY6y4tWjuTm3fK3l5e/UrA+8SR+PX0CWm2puADR7eNyvWK1YmogNOf//FA== + template: + metadata: + annotations: + kustomize.toolkit.fluxcd.io/reconcile: disabled + sealedsecrets.bitnami.com/managed: "true"