From f39c967c64407aabbad6151c7b42eae664f56c35 Mon Sep 17 00:00:00 2001 From: Build Tools Date: Wed, 5 Feb 2025 16:36:46 +0000 Subject: [PATCH 1/8] maybe this works now :sob: --- bin/conda_utils.sh | 23 ++++++++++++++++++----- bin/run-ansible.sh | 9 ++++++++- 2 files changed, 26 insertions(+), 6 deletions(-) diff --git a/bin/conda_utils.sh b/bin/conda_utils.sh index dc36b9de6..3ead011d9 100644 --- a/bin/conda_utils.sh +++ b/bin/conda_utils.sh @@ -18,8 +18,8 @@ conda_ws_name="aurora_conda_ws" miniconda_install_root="${HOME}/.shadow_miniconda" miniconda_install_location="${miniconda_install_root}/miniconda" miniconda_installer="${miniconda_install_root}/miniconda_installer.sh" -miniconda_installer_url="https://repo.anaconda.com/miniconda/Miniconda3-py311_23.5.2-0-Linux-x86_64.sh" -miniconda_checksum="634d76df5e489c44ade4085552b97bebc786d49245ed1a830022b0b406de5817" +miniconda_installer_url="https://repo.anaconda.com/miniconda/Miniconda3-py38_23.11.0-1-Linux-x86_64.sh" +miniconda_checksum="ad3cb53ddfbadd190172b864337572206733ae75515fcfb17157cc8f2cb907a5" packages_download_root="${miniconda_install_root}/aurora_host_packages" # Some molecule tests install to `/home/...` (no user account) @@ -67,7 +67,17 @@ _fetch_new_files() { echo "Fetching ${aws_bucket_dir}..." mkdir -p $local_download_dir - + alias xq="/home/tom/.shadow_miniconda/miniconda/bin/xq" + x1=$(curl -Ls ${aws_bucket_url}) + x2=$(echo $x1 | xq) + x3=$(echo $x2 | grep $aws_bucket_dir) + x4=$(echo $x3 | grep 'Key') + x5=$(echo $x4 | sed -r "s/.*${aws_bucket_dir}\///g" ) + echo "x1: $x1" + echo "x2: $x2" + echo "x3: $x3" + echo "x4: $x4" + echo "x5: $x5" remote_packages=$(curl -Ls ${aws_bucket_url} | xq | grep $aws_bucket_dir | grep 'Key' | sed -r "s/.*${aws_bucket_dir}\///g" | sed -r 's/",//g' | sed -r 's;;;g') echo "remote_packages: ${remote_packages}" @@ -116,14 +126,17 @@ create_conda_ws(){ if [ -d "$shadow_conda_ws_dir" ]; then rm -rf $shadow_conda_ws_dir fi + ${miniconda_install_location}/bin/conda create -y -n ${conda_ws_name} python=3.8 && source ${miniconda_install_location}/bin/activate ${conda_ws_name} - python -m pip install yq xq + # conda env config vars set PYTHONNOUSERSITE=1 + # conda activate ${conda_ws_name} + ${miniconda_install_location}/bin/pip3 install yq xq } fetch_pip_files(){ _fetch_new_files "http://shadowrobot.aurora-host-packages-${codename}.s3.eu-west-2.amazonaws.com" "pip_packages"; } fetch_ansible_files() { _fetch_new_files "http://shadowrobot.aurora-host-packages-${codename}.s3.eu-west-2.amazonaws.com" "ansible_collections"; } -install_pip_packages() { ANSIBLE_SKIP_CONFLICT_CHECK=1 python -m pip install ${packages_download_root}/pip_packages/* ; } +install_pip_packages() { ANSIBLE_SKIP_CONFLICT_CHECK=1 ${miniconda_install_location}/bin/pip3 install ${packages_download_root}/pip_packages/* ; } install_ansible_collections() { ansible_galaxy_executable=$1 "${ansible_galaxy_executable}" collection install $(realpath ${packages_download_root}/ansible_collections/*) diff --git a/bin/run-ansible.sh b/bin/run-ansible.sh index 0f3d5068f..a26b5a321 100755 --- a/bin/run-ansible.sh +++ b/bin/run-ansible.sh @@ -357,7 +357,9 @@ echo "" pushd $aurora_home +export PYTHONNOUSERSITE=1 source $aurora_home/bin/conda_utils.sh + create_conda_ws fetch_pip_files @@ -433,6 +435,11 @@ ansible_galaxy_executable=~/.local/bin/ansible-galaxy if [[ ! -f "${ansible_galaxy_executable}" ]]; then ansible_galaxy_executable=ansible-galaxy fi +ansible_executable="${miniconda_install_location}/bin/ansible-playbook" +ansible_basic_executable="${miniconda_install_location}/bin/ansible" +ansible_galaxy_executable="${miniconda_install_location}/bin/ansible-galaxy" + +export PYTHONPATH="${shadow_conda_ws_dir}/lib/python${aurora_python_version}/site-packages" # install ansible galaxy docker and aws collections "${ansible_basic_executable}" --version @@ -460,4 +467,4 @@ echo "" echo " ------------------------------------------------" echo " | Operation completed |" echo " ------------------------------------------------" -echo "" \ No newline at end of file +echo "" From 0b9ee61e8e6fc0e1e6a95cfb79c78a2c4f960562 Mon Sep 17 00:00:00 2001 From: Build Tools Date: Wed, 5 Feb 2025 16:48:13 +0000 Subject: [PATCH 2/8] plz find yq --- bin/run-ansible.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/bin/run-ansible.sh b/bin/run-ansible.sh index a26b5a321..fee1bf04e 100755 --- a/bin/run-ansible.sh +++ b/bin/run-ansible.sh @@ -361,7 +361,7 @@ export PYTHONNOUSERSITE=1 source $aurora_home/bin/conda_utils.sh create_conda_ws - +export PYTHONPATH="${shadow_conda_ws_dir}/lib/python${aurora_python_version}/site-packages:${miniconda_install_location}/bin" fetch_pip_files fetch_ansible_files @@ -439,7 +439,6 @@ ansible_executable="${miniconda_install_location}/bin/ansible-playbook" ansible_basic_executable="${miniconda_install_location}/bin/ansible" ansible_galaxy_executable="${miniconda_install_location}/bin/ansible-galaxy" -export PYTHONPATH="${shadow_conda_ws_dir}/lib/python${aurora_python_version}/site-packages" # install ansible galaxy docker and aws collections "${ansible_basic_executable}" --version From 13e93c81e89b62c846b99d31a8a99b982eafcabc Mon Sep 17 00:00:00 2001 From: Build Tools Date: Wed, 5 Feb 2025 16:52:09 +0000 Subject: [PATCH 3/8] testing --- bin/conda_utils.sh | 2 +- bin/run-ansible.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/bin/conda_utils.sh b/bin/conda_utils.sh index 3ead011d9..fcf08f432 100644 --- a/bin/conda_utils.sh +++ b/bin/conda_utils.sh @@ -64,7 +64,7 @@ _fetch_new_files() { aws_bucket_url=$1 aws_bucket_dir=$2 local_download_dir="${packages_download_root}/${aws_bucket_dir}" - + export PYTHONPATH="${shadow_conda_ws_dir}/lib/python${aurora_python_version}/site-packages:${miniconda_install_location}/bin" echo "Fetching ${aws_bucket_dir}..." mkdir -p $local_download_dir alias xq="/home/tom/.shadow_miniconda/miniconda/bin/xq" diff --git a/bin/run-ansible.sh b/bin/run-ansible.sh index fee1bf04e..e24f2e845 100755 --- a/bin/run-ansible.sh +++ b/bin/run-ansible.sh @@ -359,7 +359,7 @@ pushd $aurora_home export PYTHONNOUSERSITE=1 source $aurora_home/bin/conda_utils.sh - +export PYTHONNOUSERSITE=1 create_conda_ws export PYTHONPATH="${shadow_conda_ws_dir}/lib/python${aurora_python_version}/site-packages:${miniconda_install_location}/bin" fetch_pip_files From 4e1793d880b17b8b40ad1d3d05d99fc2397535dd Mon Sep 17 00:00:00 2001 From: Build Tools Date: Wed, 5 Feb 2025 16:57:04 +0000 Subject: [PATCH 4/8] testing --- bin/conda_utils.sh | 3 ++- bin/run-ansible.sh | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/bin/conda_utils.sh b/bin/conda_utils.sh index fcf08f432..1147d5c8e 100644 --- a/bin/conda_utils.sh +++ b/bin/conda_utils.sh @@ -15,6 +15,7 @@ # with this program. If not, see . conda_ws_name="aurora_conda_ws" +aurora_python_version="3.8" miniconda_install_root="${HOME}/.shadow_miniconda" miniconda_install_location="${miniconda_install_root}/miniconda" miniconda_installer="${miniconda_install_root}/miniconda_installer.sh" @@ -64,7 +65,7 @@ _fetch_new_files() { aws_bucket_url=$1 aws_bucket_dir=$2 local_download_dir="${packages_download_root}/${aws_bucket_dir}" - export PYTHONPATH="${shadow_conda_ws_dir}/lib/python${aurora_python_version}/site-packages:${miniconda_install_location}/bin" + export PYTHONPATH="${miniconda_install_location}/lib/python3.8/site-packages:${miniconda_install_location}/bin" echo "Fetching ${aws_bucket_dir}..." mkdir -p $local_download_dir alias xq="/home/tom/.shadow_miniconda/miniconda/bin/xq" diff --git a/bin/run-ansible.sh b/bin/run-ansible.sh index e24f2e845..527ecce6f 100755 --- a/bin/run-ansible.sh +++ b/bin/run-ansible.sh @@ -342,7 +342,7 @@ while sudo fuser /var/lib/dpkg/lock >/dev/null 2>&1; do done # jq is needed for yq, which installs xq, which helps parse aws s3 http requests -sudo apt-get install -y git jq curl lsb-release libyaml-dev libssl-dev libffi-dev sshpass +# sudo apt-get install -y git jq curl lsb-release libyaml-dev libssl-dev libffi-dev sshpass sudo chown $USER:$USER $aurora_home || true sudo rm -rf ${aurora_home} From 68e1bfcb4a6d5f354fbf03e72b81d6b033b50879 Mon Sep 17 00:00:00 2001 From: Build Tools Date: Wed, 5 Feb 2025 17:01:15 +0000 Subject: [PATCH 5/8] testing --- bin/conda_utils.sh | 17 ++--------------- bin/run-ansible.sh | 14 ++++++-------- 2 files changed, 8 insertions(+), 23 deletions(-) diff --git a/bin/conda_utils.sh b/bin/conda_utils.sh index 1147d5c8e..1d88aca48 100644 --- a/bin/conda_utils.sh +++ b/bin/conda_utils.sh @@ -65,20 +65,10 @@ _fetch_new_files() { aws_bucket_url=$1 aws_bucket_dir=$2 local_download_dir="${packages_download_root}/${aws_bucket_dir}" - export PYTHONPATH="${miniconda_install_location}/lib/python3.8/site-packages:${miniconda_install_location}/bin" + PYTHONPATH="${miniconda_install_location}/lib/python3.8/site-packages:${miniconda_install_location}/bin" echo "Fetching ${aws_bucket_dir}..." mkdir -p $local_download_dir - alias xq="/home/tom/.shadow_miniconda/miniconda/bin/xq" - x1=$(curl -Ls ${aws_bucket_url}) - x2=$(echo $x1 | xq) - x3=$(echo $x2 | grep $aws_bucket_dir) - x4=$(echo $x3 | grep 'Key') - x5=$(echo $x4 | sed -r "s/.*${aws_bucket_dir}\///g" ) - echo "x1: $x1" - echo "x2: $x2" - echo "x3: $x3" - echo "x4: $x4" - echo "x5: $x5" + # alias xq="/home/tom/.shadow_miniconda/miniconda/bin/xq" remote_packages=$(curl -Ls ${aws_bucket_url} | xq | grep $aws_bucket_dir | grep 'Key' | sed -r "s/.*${aws_bucket_dir}\///g" | sed -r 's/",//g' | sed -r 's;;;g') echo "remote_packages: ${remote_packages}" @@ -127,10 +117,7 @@ create_conda_ws(){ if [ -d "$shadow_conda_ws_dir" ]; then rm -rf $shadow_conda_ws_dir fi - ${miniconda_install_location}/bin/conda create -y -n ${conda_ws_name} python=3.8 && source ${miniconda_install_location}/bin/activate ${conda_ws_name} - # conda env config vars set PYTHONNOUSERSITE=1 - # conda activate ${conda_ws_name} ${miniconda_install_location}/bin/pip3 install yq xq } diff --git a/bin/run-ansible.sh b/bin/run-ansible.sh index 527ecce6f..fb0253a39 100755 --- a/bin/run-ansible.sh +++ b/bin/run-ansible.sh @@ -359,9 +359,10 @@ pushd $aurora_home export PYTHONNOUSERSITE=1 source $aurora_home/bin/conda_utils.sh -export PYTHONNOUSERSITE=1 +PYTHONPATH="${miniconda_install_location}/lib/python3.8/site-packages:${miniconda_install_location}/bin" create_conda_ws -export PYTHONPATH="${shadow_conda_ws_dir}/lib/python${aurora_python_version}/site-packages:${miniconda_install_location}/bin" +# PYTHONPATH="${miniconda_install_location}/lib/python3.8/site-packages:${miniconda_install_location}/bin" +# PYTHONPATH="${shadow_conda_ws_dir}/lib/python${aurora_python_version}/site-packages:${miniconda_install_location}/bin" fetch_pip_files fetch_ansible_files @@ -423,21 +424,18 @@ else echo "" fi -ansible_executable=~/.local/bin/ansible-playbook +ansible_executable="${miniconda_install_location}/bin/ansible-playbook" if [[ ! -f "${ansible_executable}" ]]; then ansible_executable=ansible-playbook fi -ansible_basic_executable=~/.local/bin/ansible +ansible_basic_executable="${miniconda_install_location}/bin/ansible" if [[ ! -f "${ansible_basic_executable}" ]]; then ansible_basic_executable=ansible fi -ansible_galaxy_executable=~/.local/bin/ansible-galaxy +ansible_galaxy_executable="${miniconda_install_location}/bin/ansible-galaxy" if [[ ! -f "${ansible_galaxy_executable}" ]]; then ansible_galaxy_executable=ansible-galaxy fi -ansible_executable="${miniconda_install_location}/bin/ansible-playbook" -ansible_basic_executable="${miniconda_install_location}/bin/ansible" -ansible_galaxy_executable="${miniconda_install_location}/bin/ansible-galaxy" # install ansible galaxy docker and aws collections From 76e7f05729a73eccdfea9af64f557b07bfdbe2ae Mon Sep 17 00:00:00 2001 From: Build Tools Date: Wed, 5 Feb 2025 17:08:47 +0000 Subject: [PATCH 6/8] testing --- bin/conda_utils.sh | 2 +- bin/run-ansible.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/bin/conda_utils.sh b/bin/conda_utils.sh index 1d88aca48..bfade9862 100644 --- a/bin/conda_utils.sh +++ b/bin/conda_utils.sh @@ -65,7 +65,7 @@ _fetch_new_files() { aws_bucket_url=$1 aws_bucket_dir=$2 local_download_dir="${packages_download_root}/${aws_bucket_dir}" - PYTHONPATH="${miniconda_install_location}/lib/python3.8/site-packages:${miniconda_install_location}/bin" + # PYTHONPATH="${miniconda_install_location}/lib/python3.8/site-packages:${miniconda_install_location}/bin" echo "Fetching ${aws_bucket_dir}..." mkdir -p $local_download_dir # alias xq="/home/tom/.shadow_miniconda/miniconda/bin/xq" diff --git a/bin/run-ansible.sh b/bin/run-ansible.sh index fb0253a39..406f64fd0 100755 --- a/bin/run-ansible.sh +++ b/bin/run-ansible.sh @@ -359,7 +359,7 @@ pushd $aurora_home export PYTHONNOUSERSITE=1 source $aurora_home/bin/conda_utils.sh -PYTHONPATH="${miniconda_install_location}/lib/python3.8/site-packages:${miniconda_install_location}/bin" +export PYTHONPATH="${miniconda_install_location}/lib/python3.8/site-packages:${miniconda_install_location}/bin" create_conda_ws # PYTHONPATH="${miniconda_install_location}/lib/python3.8/site-packages:${miniconda_install_location}/bin" # PYTHONPATH="${shadow_conda_ws_dir}/lib/python${aurora_python_version}/site-packages:${miniconda_install_location}/bin" From d3231c69e896fa65f1903bd81b15de1b4edfc724 Mon Sep 17 00:00:00 2001 From: Build Tools Date: Wed, 5 Feb 2025 17:10:26 +0000 Subject: [PATCH 7/8] seems to work --- bin/conda_utils.sh | 5 ++--- bin/run-ansible.sh | 6 +++--- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/bin/conda_utils.sh b/bin/conda_utils.sh index bfade9862..199c681e5 100644 --- a/bin/conda_utils.sh +++ b/bin/conda_utils.sh @@ -15,7 +15,6 @@ # with this program. If not, see . conda_ws_name="aurora_conda_ws" -aurora_python_version="3.8" miniconda_install_root="${HOME}/.shadow_miniconda" miniconda_install_location="${miniconda_install_root}/miniconda" miniconda_installer="${miniconda_install_root}/miniconda_installer.sh" @@ -65,10 +64,10 @@ _fetch_new_files() { aws_bucket_url=$1 aws_bucket_dir=$2 local_download_dir="${packages_download_root}/${aws_bucket_dir}" - # PYTHONPATH="${miniconda_install_location}/lib/python3.8/site-packages:${miniconda_install_location}/bin" + echo "Fetching ${aws_bucket_dir}..." mkdir -p $local_download_dir - # alias xq="/home/tom/.shadow_miniconda/miniconda/bin/xq" + remote_packages=$(curl -Ls ${aws_bucket_url} | xq | grep $aws_bucket_dir | grep 'Key' | sed -r "s/.*${aws_bucket_dir}\///g" | sed -r 's/",//g' | sed -r 's;;;g') echo "remote_packages: ${remote_packages}" diff --git a/bin/run-ansible.sh b/bin/run-ansible.sh index 406f64fd0..312367373 100755 --- a/bin/run-ansible.sh +++ b/bin/run-ansible.sh @@ -342,7 +342,7 @@ while sudo fuser /var/lib/dpkg/lock >/dev/null 2>&1; do done # jq is needed for yq, which installs xq, which helps parse aws s3 http requests -# sudo apt-get install -y git jq curl lsb-release libyaml-dev libssl-dev libffi-dev sshpass +sudo apt-get install -y git jq curl lsb-release libyaml-dev libssl-dev libffi-dev sshpass sudo chown $USER:$USER $aurora_home || true sudo rm -rf ${aurora_home} @@ -360,9 +360,9 @@ pushd $aurora_home export PYTHONNOUSERSITE=1 source $aurora_home/bin/conda_utils.sh export PYTHONPATH="${miniconda_install_location}/lib/python3.8/site-packages:${miniconda_install_location}/bin" + create_conda_ws -# PYTHONPATH="${miniconda_install_location}/lib/python3.8/site-packages:${miniconda_install_location}/bin" -# PYTHONPATH="${shadow_conda_ws_dir}/lib/python${aurora_python_version}/site-packages:${miniconda_install_location}/bin" + fetch_pip_files fetch_ansible_files From f3c7d0d80a9b2b0107e39e665b72a7afa75f6357 Mon Sep 17 00:00:00 2001 From: Build Tools Date: Wed, 5 Feb 2025 17:17:44 +0000 Subject: [PATCH 8/8] add option for password to be read from a vault file (for test machine infrastructure) --- bin/run-ansible.sh | 29 +++++++++++++++++++++++++---- 1 file changed, 25 insertions(+), 4 deletions(-) diff --git a/bin/run-ansible.sh b/bin/run-ansible.sh index 312367373..e85674c5c 100755 --- a/bin/run-ansible.sh +++ b/bin/run-ansible.sh @@ -41,6 +41,7 @@ aurora_home=/tmp/aurora playbook=$1 aurora_limit=all +test_machine=false shift while [[ $# -gt 1 ]] @@ -67,6 +68,10 @@ case ${key} in read_secure="$2" shift 2 ;; + --test-machine) + test_machine="$2" + shift 2 + ;; *) break ;; @@ -74,7 +79,7 @@ esac done -if [[ "${playbook}" = "server_and_nuc_deploy" || "${playbook}" = "teleop_deploy" ]]; then +if [[ "${playbook}" = "server_and_nuc_deploy" || "${playbook}" = "teleop_deploy" ]] && [[ "${test_machine}" = "false" ]]; then if [[ -z ${read_secure} ]]; then read_secure="sudo_password" else @@ -116,6 +121,7 @@ echo "playbook = ${playbook}" echo "branch = ${aurora_tools_branch}" echo "inventory = ${aurora_inventory}" echo "limit = ${aurora_limit}" +echo "Testing setup= ${test_machine}" export ANSIBLE_ROLES_PATH="${aurora_home}/ansible/roles" export ANSIBLE_CALLBACK_PLUGINS="${HOME}/.ansible/plugins/callback:/usr/share/ansible/plugins/callback:${aurora_home}/ansible/playbooks/callback_plugins" @@ -328,6 +334,11 @@ echo " | Installing needed packages |" echo " ---------------------------------" echo "" +if [[ "${test_machine}" = "true" ]]; then + echo $test_password | sudo -S echo "Running on testing machine. Retrieving passwords from ENV..." + formatted_extra_vars="$formatted_extra_vars sudo_password=$test_password docker_username=$docker_username docker_password=$docker_password ansible_sudo_pass=$test_password" +fi + # Wait for apt-get update lock file to be released while (sudo fuser /var/lib/apt/lists/lock >/dev/null 2>&1) || (sudo fuser /var/lib/dpkg/lock >/dev/null 2>&1) do echo "Waiting for apt-get update file lock..." @@ -392,7 +403,11 @@ if [[ "${playbook}" = "server_and_nuc_deploy" ]]; then else aurora_inventory="ansible/inventory/server_and_nuc/${aurora_inventory}" fi - ansible_flags="${ansible_flags} --ask-vault-pass" + if [[ "${test_machine}" = "true" ]]; then + ansible_flags="${ansible_flags} --vault-password-file /home/$USER/vault.sh" + else + ansible_flags="${ansible_flags} --ask-vault-pass" + fi echo "" echo " ---------------------------------------------------" echo " | VAULT password: |" @@ -400,7 +415,11 @@ if [[ "${playbook}" = "server_and_nuc_deploy" ]]; then echo " ---------------------------------------------------" echo "" elif [[ "${playbook}" = "teleop_deploy" ]]; then - ansible_flags="${ansible_flags} --ask-vault-pass" + if [[ "${test_machine}" = "true" ]]; then + ansible_flags="${ansible_flags} --vault-password-file /home/$USER/vault.sh" + else + ansible_flags="${ansible_flags} --ask-vault-pass" + fi if [[ "${aurora_inventory}" = "" ]]; then aurora_inventory="ansible/inventory/teleop/production" else @@ -415,7 +434,9 @@ elif [[ "${playbook}" = "teleop_deploy" ]]; then echo "" else aurora_inventory="ansible/inventory/${aurora_inventory}" - ansible_flags="${ansible_flags} --ask-become-pass" + if [[ "${test_machine}" != "true" ]]; then + ansible_flags="${ansible_flags} --ask-become-pass" + fi echo "" echo " --------------------------------------------" echo " | BECOME password: |"