diff --git a/k8s/production/runners/protected/graviton/3/release.yaml b/k8s/production/runners/protected/graviton/3/release.yaml index 304496765..5be568a05 100644 --- a/k8s/production/runners/protected/graviton/3/release.yaml +++ b/k8s/production/runners/protected/graviton/3/release.yaml @@ -76,6 +76,36 @@ spec: unset GITLAB_OIDC_TOKEN """ + post_build_script = """ + echo 'Executing Spack pre-build setup script' + + for cmd in "${PY3:-}" python3 python; do + if command -v > /dev/null "$cmd"; then + export PY3="$(command -v "$cmd")" + break + fi + done + + if [ -z "${PY3:-}" ]; then + echo "Unable to find python3 executable" + exit 1 + fi + + $PY3 -c "import urllib.request;urllib.request.urlretrieve('https://raw.githubusercontent.com/spack/spack-infrastructure/main/scripts/gitlab_runner_pre_build/pre_build.py', 'pre_build.py')" + $PY3 pre_build.py > envvars + + . ./envvars + rm -f envvars + unset GITLAB_OIDC_TOKEN + + echo 'Trusting GPG secrets' + k=$CI_GPG_KEY_ROOT/e4s.gpg; [[ -r $k ]] && spack gpg trust -y $k + k=$CI_GPG_KEY_ROOT/intermediate_ci_signing_key.gpg; [[ -r $k ]] && spack gpg trust -y $k + k=$CI_GPG_KEY_ROOT/spack_public_key.gpg; [[ -r $k ]] && spack gpg trust -y $k + + spack buildcache push --without-build-dependencies buildcache-destination "/${SPACK_JOB_SPEC_DAG_HASH}" + """ + output_limit = 20480 environment = ["FF_GITLAB_REGISTRY_HELPER_IMAGE=1"] [runners.kubernetes] diff --git a/k8s/production/runners/protected/graviton/4/release.yaml b/k8s/production/runners/protected/graviton/4/release.yaml index ef5db3666..beca2dd83 100644 --- a/k8s/production/runners/protected/graviton/4/release.yaml +++ b/k8s/production/runners/protected/graviton/4/release.yaml @@ -76,6 +76,36 @@ spec: unset GITLAB_OIDC_TOKEN """ + post_build_script = """ + echo 'Executing Spack pre-build setup script' + + for cmd in "${PY3:-}" python3 python; do + if command -v > /dev/null "$cmd"; then + export PY3="$(command -v "$cmd")" + break + fi + done + + if [ -z "${PY3:-}" ]; then + echo "Unable to find python3 executable" + exit 1 + fi + + $PY3 -c "import urllib.request;urllib.request.urlretrieve('https://raw.githubusercontent.com/spack/spack-infrastructure/main/scripts/gitlab_runner_pre_build/pre_build.py', 'pre_build.py')" + $PY3 pre_build.py > envvars + + . ./envvars + rm -f envvars + unset GITLAB_OIDC_TOKEN + + echo 'Trusting GPG secrets' + k=$CI_GPG_KEY_ROOT/e4s.gpg; [[ -r $k ]] && spack gpg trust -y $k + k=$CI_GPG_KEY_ROOT/intermediate_ci_signing_key.gpg; [[ -r $k ]] && spack gpg trust -y $k + k=$CI_GPG_KEY_ROOT/spack_public_key.gpg; [[ -r $k ]] && spack gpg trust -y $k + + spack buildcache push --without-build-dependencies buildcache-destination "/${SPACK_JOB_SPEC_DAG_HASH}" + """ + output_limit = 20480 environment = ["FF_GITLAB_REGISTRY_HELPER_IMAGE=1"] [runners.kubernetes] diff --git a/k8s/production/runners/protected/x86_64/v2-win/release.yaml b/k8s/production/runners/protected/x86_64/v2-win/release.yaml index d56e56abe..20bc5a773 100644 --- a/k8s/production/runners/protected/x86_64/v2-win/release.yaml +++ b/k8s/production/runners/protected/x86_64/v2-win/release.yaml @@ -86,6 +86,40 @@ spec: $env:GITLAB_OIDC_TOKEN = $null """ + post_build_script = """ + Write-Output 'Executing Spack pre-build setup script' + + $py=(get-command -ErrorAction SilentlyContinue python) + if ( -not $py ) { + Write-Output 'Python was not found on the system. Add it to the PATH or install.' + exit 1 + } + + $scriptPath = Join-Path $HOME 'pre_build.py' + + $wc = New-Object System.Net.WebClient + $wc.DownloadFile('https://raw.githubusercontent.com/spack/spack-infrastructure/main/scripts/gitlab_runner_pre_build/pre_build.py', $scriptPath) + + & python $scriptPath | Out-File -FilePath 'envvars.ps1' -Encoding utf8 + if ($LASTEXITCODE -ne 0) { + Write-Output "Prebuild script pre_build.py failed with exit code $LASTEXITCODE. See error above for more info" + exit 1 + } + + ./envvars.ps1 + + Remove-Item $scriptPath -Force + Remove-Item 'envvars.ps1' -Force + + $env:GITLAB_OIDC_TOKEN = $null + + Write-Output 'Trusting GPG secrets' + If (Test-Path -path C:\\key\intermediate_ci_signing_key.gpg) { spack.ps1 gpg trust -y C:\\key\intermediate_ci_signing_key.gpg } + If (Test-Path -path C:\\key\spack_public_key.gpg) { spack.ps1 gpg trust -y C:\\key\spack_public_key.gpg } + + spack.ps1 buildcache push --without-build-dependencies buildcache-destination "/${SPACK_JOB_SPEC_DAG_HASH}" + """ + shell = "powershell" executor = "kubernetes" output_limit = 20480 diff --git a/k8s/production/runners/protected/x86_64/v2/release.yaml b/k8s/production/runners/protected/x86_64/v2/release.yaml index 39501049b..2f24def63 100644 --- a/k8s/production/runners/protected/x86_64/v2/release.yaml +++ b/k8s/production/runners/protected/x86_64/v2/release.yaml @@ -76,6 +76,36 @@ spec: unset GITLAB_OIDC_TOKEN """ + post_build_script = """ + echo 'Executing Spack pre-build setup script' + + for cmd in "${PY3:-}" python3 python; do + if command -v > /dev/null "$cmd"; then + export PY3="$(command -v "$cmd")" + break + fi + done + + if [ -z "${PY3:-}" ]; then + echo "Unable to find python3 executable" + exit 1 + fi + + $PY3 -c "import urllib.request;urllib.request.urlretrieve('https://raw.githubusercontent.com/spack/spack-infrastructure/main/scripts/gitlab_runner_pre_build/pre_build.py', 'pre_build.py')" + $PY3 pre_build.py > envvars + + . ./envvars + rm -f envvars + unset GITLAB_OIDC_TOKEN + + echo 'Trusting GPG secrets' + k=$CI_GPG_KEY_ROOT/e4s.gpg; [[ -r $k ]] && spack gpg trust -y $k + k=$CI_GPG_KEY_ROOT/intermediate_ci_signing_key.gpg; [[ -r $k ]] && spack gpg trust -y $k + k=$CI_GPG_KEY_ROOT/spack_public_key.gpg; [[ -r $k ]] && spack gpg trust -y $k + + spack buildcache push --without-build-dependencies buildcache-destination "/${SPACK_JOB_SPEC_DAG_HASH}" + """ + output_limit = 20480 environment = ["FF_GITLAB_REGISTRY_HELPER_IMAGE=1"] [runners.kubernetes] diff --git a/k8s/production/runners/protected/x86_64/v3/release.yaml b/k8s/production/runners/protected/x86_64/v3/release.yaml index 8ce36abca..b08e0617a 100644 --- a/k8s/production/runners/protected/x86_64/v3/release.yaml +++ b/k8s/production/runners/protected/x86_64/v3/release.yaml @@ -76,6 +76,36 @@ spec: unset GITLAB_OIDC_TOKEN """ + post_build_script = """ + echo 'Executing Spack pre-build setup script' + + for cmd in "${PY3:-}" python3 python; do + if command -v > /dev/null "$cmd"; then + export PY3="$(command -v "$cmd")" + break + fi + done + + if [ -z "${PY3:-}" ]; then + echo "Unable to find python3 executable" + exit 1 + fi + + $PY3 -c "import urllib.request;urllib.request.urlretrieve('https://raw.githubusercontent.com/spack/spack-infrastructure/main/scripts/gitlab_runner_pre_build/pre_build.py', 'pre_build.py')" + $PY3 pre_build.py > envvars + + . ./envvars + rm -f envvars + unset GITLAB_OIDC_TOKEN + + echo 'Trusting GPG secrets' + k=$CI_GPG_KEY_ROOT/e4s.gpg; [[ -r $k ]] && spack gpg trust -y $k + k=$CI_GPG_KEY_ROOT/intermediate_ci_signing_key.gpg; [[ -r $k ]] && spack gpg trust -y $k + k=$CI_GPG_KEY_ROOT/spack_public_key.gpg; [[ -r $k ]] && spack gpg trust -y $k + + spack buildcache push --without-build-dependencies buildcache-destination "/${SPACK_JOB_SPEC_DAG_HASH}" + """ + output_limit = 20480 environment = ["FF_GITLAB_REGISTRY_HELPER_IMAGE=1"] [runners.kubernetes] diff --git a/k8s/production/runners/protected/x86_64/v4/release.yaml b/k8s/production/runners/protected/x86_64/v4/release.yaml index 44a99c396..58f37ba1d 100644 --- a/k8s/production/runners/protected/x86_64/v4/release.yaml +++ b/k8s/production/runners/protected/x86_64/v4/release.yaml @@ -76,6 +76,36 @@ spec: unset GITLAB_OIDC_TOKEN """ + post_build_script = """ + echo 'Executing Spack pre-build setup script' + + for cmd in "${PY3:-}" python3 python; do + if command -v > /dev/null "$cmd"; then + export PY3="$(command -v "$cmd")" + break + fi + done + + if [ -z "${PY3:-}" ]; then + echo "Unable to find python3 executable" + exit 1 + fi + + $PY3 -c "import urllib.request;urllib.request.urlretrieve('https://raw.githubusercontent.com/spack/spack-infrastructure/main/scripts/gitlab_runner_pre_build/pre_build.py', 'pre_build.py')" + $PY3 pre_build.py > envvars + + . ./envvars + rm -f envvars + unset GITLAB_OIDC_TOKEN + + echo 'Trusting GPG secrets' + k=$CI_GPG_KEY_ROOT/e4s.gpg; [[ -r $k ]] && spack gpg trust -y $k + k=$CI_GPG_KEY_ROOT/intermediate_ci_signing_key.gpg; [[ -r $k ]] && spack gpg trust -y $k + k=$CI_GPG_KEY_ROOT/spack_public_key.gpg; [[ -r $k ]] && spack gpg trust -y $k + + spack buildcache push --without-build-dependencies buildcache-destination "/${SPACK_JOB_SPEC_DAG_HASH}" + """ + output_limit = 20480 environment = ["FF_GITLAB_REGISTRY_HELPER_IMAGE=1"] [runners.kubernetes] diff --git a/k8s/production/runners/public/graviton/3/release.yaml b/k8s/production/runners/public/graviton/3/release.yaml index 6bdfdb56c..e6b343a63 100644 --- a/k8s/production/runners/public/graviton/3/release.yaml +++ b/k8s/production/runners/public/graviton/3/release.yaml @@ -76,6 +76,31 @@ spec: unset GITLAB_OIDC_TOKEN """ + post_build_script = """ + echo 'Executing Spack pre-build setup script' + + for cmd in "${PY3:-}" python3 python; do + if command -v > /dev/null "$cmd"; then + export PY3="$(command -v "$cmd")" + break + fi + done + + if [ -z "${PY3:-}" ]; then + echo "Unable to find python3 executable" + exit 1 + fi + + $PY3 -c "import urllib.request;urllib.request.urlretrieve('https://raw.githubusercontent.com/spack/spack-infrastructure/main/scripts/gitlab_runner_pre_build/pre_build.py', 'pre_build.py')" + $PY3 pre_build.py > envvars + + . ./envvars + rm -f envvars + unset GITLAB_OIDC_TOKEN + + spack buildcache push --without-build-dependencies buildcache-destination "/${SPACK_JOB_SPEC_DAG_HASH}" + """ + output_limit = 20480 environment = ["FF_GITLAB_REGISTRY_HELPER_IMAGE=1"] [runners.kubernetes] diff --git a/k8s/production/runners/public/graviton/4/release.yaml b/k8s/production/runners/public/graviton/4/release.yaml index e6251a2c0..e21f8f6f3 100644 --- a/k8s/production/runners/public/graviton/4/release.yaml +++ b/k8s/production/runners/public/graviton/4/release.yaml @@ -76,6 +76,31 @@ spec: unset GITLAB_OIDC_TOKEN """ + post_build_script = """ + echo 'Executing Spack pre-build setup script' + + for cmd in "${PY3:-}" python3 python; do + if command -v > /dev/null "$cmd"; then + export PY3="$(command -v "$cmd")" + break + fi + done + + if [ -z "${PY3:-}" ]; then + echo "Unable to find python3 executable" + exit 1 + fi + + $PY3 -c "import urllib.request;urllib.request.urlretrieve('https://raw.githubusercontent.com/spack/spack-infrastructure/main/scripts/gitlab_runner_pre_build/pre_build.py', 'pre_build.py')" + $PY3 pre_build.py > envvars + + . ./envvars + rm -f envvars + unset GITLAB_OIDC_TOKEN + + spack buildcache push --without-build-dependencies buildcache-destination "/${SPACK_JOB_SPEC_DAG_HASH}" + """ + output_limit = 20480 environment = ["FF_GITLAB_REGISTRY_HELPER_IMAGE=1"] [runners.kubernetes] diff --git a/k8s/production/runners/public/x86_64/v2-win/release.yaml b/k8s/production/runners/public/x86_64/v2-win/release.yaml index b7192b30c..4811c79a6 100644 --- a/k8s/production/runners/public/x86_64/v2-win/release.yaml +++ b/k8s/production/runners/public/x86_64/v2-win/release.yaml @@ -87,6 +87,31 @@ spec: $env:GITLAB_OIDC_TOKEN = $null """ + post_build_script = """ + echo 'Executing Spack pre-build setup script' + + for cmd in "${PY3:-}" python3 python; do + if command -v > /dev/null "$cmd"; then + export PY3="$(command -v "$cmd")" + break + fi + done + + if [ -z "${PY3:-}" ]; then + echo "Unable to find python3 executable" + exit 1 + fi + + $PY3 -c "import urllib.request;urllib.request.urlretrieve('https://raw.githubusercontent.com/spack/spack-infrastructure/main/scripts/gitlab_runner_pre_build/pre_build.py', 'pre_build.py')" + $PY3 pre_build.py > envvars + + . ./envvars + rm -f envvars + unset GITLAB_OIDC_TOKEN + + spack buildcache push --without-build-dependencies buildcache-destination "/${SPACK_JOB_SPEC_DAG_HASH}" + """ + shell = "powershell" executor = "kubernetes" output_limit = 20480 diff --git a/k8s/production/runners/public/x86_64/v2/release.yaml b/k8s/production/runners/public/x86_64/v2/release.yaml index a756873f5..9fd89c369 100644 --- a/k8s/production/runners/public/x86_64/v2/release.yaml +++ b/k8s/production/runners/public/x86_64/v2/release.yaml @@ -78,6 +78,31 @@ spec: fi """ + post_build_script = """ + echo 'Executing Spack pre-build setup script' + + for cmd in "${PY3:-}" python3 python; do + if command -v > /dev/null "$cmd"; then + export PY3="$(command -v "$cmd")" + break + fi + done + + if [ -z "${PY3:-}" ]; then + echo "Unable to find python3 executable" + exit 1 + fi + + $PY3 -c "import urllib.request;urllib.request.urlretrieve('https://raw.githubusercontent.com/spack/spack-infrastructure/main/scripts/gitlab_runner_pre_build/pre_build.py', 'pre_build.py')" + $PY3 pre_build.py > envvars + + . ./envvars + rm -f envvars + unset GITLAB_OIDC_TOKEN + + spack buildcache push --without-build-dependencies buildcache-destination "/${SPACK_JOB_SPEC_DAG_HASH}" + """ + output_limit = 20480 environment = ["FF_GITLAB_REGISTRY_HELPER_IMAGE=1"] [runners.kubernetes] diff --git a/k8s/production/runners/public/x86_64/v3/release.yaml b/k8s/production/runners/public/x86_64/v3/release.yaml index 7a615a54b..ee896fdc9 100644 --- a/k8s/production/runners/public/x86_64/v3/release.yaml +++ b/k8s/production/runners/public/x86_64/v3/release.yaml @@ -76,6 +76,31 @@ spec: unset GITLAB_OIDC_TOKEN """ + post_build_script = """ + echo 'Executing Spack pre-build setup script' + + for cmd in "${PY3:-}" python3 python; do + if command -v > /dev/null "$cmd"; then + export PY3="$(command -v "$cmd")" + break + fi + done + + if [ -z "${PY3:-}" ]; then + echo "Unable to find python3 executable" + exit 1 + fi + + $PY3 -c "import urllib.request;urllib.request.urlretrieve('https://raw.githubusercontent.com/spack/spack-infrastructure/main/scripts/gitlab_runner_pre_build/pre_build.py', 'pre_build.py')" + $PY3 pre_build.py > envvars + + . ./envvars + rm -f envvars + unset GITLAB_OIDC_TOKEN + + spack buildcache push --without-build-dependencies "/${SPACK_JOB_SPEC_DAG_HASH}" + """ + output_limit = 20480 environment = ["FF_GITLAB_REGISTRY_HELPER_IMAGE=1"] [runners.kubernetes] diff --git a/k8s/production/runners/public/x86_64/v4/release.yaml b/k8s/production/runners/public/x86_64/v4/release.yaml index 948d2f836..715c18dc8 100644 --- a/k8s/production/runners/public/x86_64/v4/release.yaml +++ b/k8s/production/runners/public/x86_64/v4/release.yaml @@ -76,6 +76,31 @@ spec: unset GITLAB_OIDC_TOKEN """ + post_build_script = """ + echo 'Executing Spack pre-build setup script' + + for cmd in "${PY3:-}" python3 python; do + if command -v > /dev/null "$cmd"; then + export PY3="$(command -v "$cmd")" + break + fi + done + + if [ -z "${PY3:-}" ]; then + echo "Unable to find python3 executable" + exit 1 + fi + + $PY3 -c "import urllib.request;urllib.request.urlretrieve('https://raw.githubusercontent.com/spack/spack-infrastructure/main/scripts/gitlab_runner_pre_build/pre_build.py', 'pre_build.py')" + $PY3 pre_build.py > envvars + + . ./envvars + rm -f envvars + unset GITLAB_OIDC_TOKEN + + spack buildcache push --without-build-dependencies buildcache-destination "/${SPACK_JOB_SPEC_DAG_HASH}" + """ + output_limit = 20480 environment = ["FF_GITLAB_REGISTRY_HELPER_IMAGE=1"] [runners.kubernetes] diff --git a/scripts/gitlab_runner_pre_build/pre_build.py b/scripts/gitlab_runner_pre_build/pre_build.py index eac27ed3b..abab8f462 100644 --- a/scripts/gitlab_runner_pre_build/pre_build.py +++ b/scripts/gitlab_runner_pre_build/pre_build.py @@ -114,6 +114,12 @@ def _gitlab_token_to_credentials(gitlab_token): print("GITLAB_OIDC_TOKEN not in the environment", file=sys.stderr) sys.exit(0) # this isn't an error yet. + env_vars = ["AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN"] + + if all(var in os.environ for var in env_vars): + print("AWS credentials already set.") + sys.exit(0) + response = _gitlab_token_to_credentials(os.environ["GITLAB_OIDC_TOKEN"]) set_env_cmd = "export "