General improvement

.gitignore

@@ -0,0 +1,2 @@
+images/*
+terraform/**

README.rst

@@ -56,34 +56,11 @@ Prerequisites
 - Terraform (>=1.3.6)
 
 From within the MNAIOv2 directory, install Ansible and required collections
-with the following commands:
+and terraform with the following command:
 
 .. code-block:: bash
 
-    Ubuntu 20.04 LTS
-    ----------------
-    sudo apt install python3-pip
-    sudo pip3 install ansible-core==2.13.5
-    ansible-galaxy collection install -r requirements.yml
-
-    MacOS X (Homebrew)
-    ------------------
-    brew install ansible
-    ansible-galaxy collection install -r requirements.yml
-
-Install Terraform:
-
-.. code-block:: bash
-
-    Ubuntu 20.04 LTS
-    ----------------
-    curl -fsSL https://apt.releases.hashicorp.com/gpg | sudo apt-key add -
-    sudo apt-add-repository "deb [arch=amd64] https://apt.releases.hashicorp.com $(lsb_release -cs) main"
-    sudo apt-get update && sudo apt-get install terraform=1.3.6
-
-    MacOS X (Homebrew)
-    ------------------
-    brew install terraform
+    scripts/bootstrap.sh
 
 Overrides
 ---------
@@ -207,7 +184,8 @@ To SSH to the deploy node, use the private key. Attach to the existing tmux sess
 
 .. code-block:: bash
 
-    jdenton@MBP-M1 % ssh -i id_rsa_mnaio.key ubuntu@192.168.2.183
+    % ssh -i id_rsa_mnaio.key ubuntu@192.168.2.183
+
     Welcome to Ubuntu 20.04.5 LTS (GNU/Linux 5.4.0-135-generic x86_64)
 
     * Documentation:  https://help.ubuntu.com

ansible-env.rc

@@ -16,3 +16,7 @@ export ANSIBLE_SSH_ARGS="-o ControlMaster=no \
   -o VerifyHostKeyDNS=no \
   -o ForwardX11=no \
   -o ForwardAgent=yes"
+
+export ANSIBLE_COLLECTIONS_PATH="/opt/ansible-mnaio2/"
+
+export PATH="/opt/ansible-mnaio2/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:${PATH}"

build.sh

@@ -31,6 +31,8 @@ then
     exit 1
 fi
 
+export MNAIO_TF_STATE_FILE=${MNAIO_TF_STATE_FILE:-"terraform.tfstate"}
+
 # DO NOT MODIFY #
 export TF_VAR_image=${MNAIO_OSA_VM_IMAGE:-"focal"}
 export MNAIO_DEPLOY=${MNAIO_DEPLOY:-"osa"}
@@ -41,10 +43,10 @@ export MNAIO_OSA_VM_IMAGE_UUID=${MNAIO_OSA_VM_IMAGE_UUID:-""}
 ###################
 
 pushd terraform
-terraform init
+    terraform init
 popd
 
-# This playbook downloads images to the local machine for later uploading to Glance
+# This playbook downloads images to the local machine for later upload into glance
 ansible-playbook playbooks/download-images.yml \
    -e osa_vm_image=${MNAIO_OSA_VM_IMAGE:-"focal"}
 

destroy.sh

@@ -18,11 +18,10 @@ set -u
 source ansible-env.rc
 
 # Destroy the Terraform environment
-
 pushd terraform
-TF_VAR_image=dummy \
-TF_VAR_external_network='{"name":"dummy","uuid":"dummy"}' \
-terraform destroy
+  TF_VAR_image=dummy \
+  TF_VAR_external_network='{"name":"dummy","uuid":"dummy"}' \
+  terraform destroy -state=${MNAIO_TF_STATE_FILE}
 popd
 
 ###############

global-requirement-pins.txt

@@ -0,0 +1,4 @@
+#
+# Limit pip requirements when necessary
+#
+

playbooks/clean.yml

@@ -25,7 +25,7 @@
     - vars/main.yml
   tasks:
     - name: Read Terraform state file
-      ansible.builtin.shell: "cat {{ playbook_dir }}/../terraform/terraform.tfstate"
+      ansible.builtin.shell: "cat {{ playbook_dir }}/../terraform/{{ tf_state_file }}"
       register: result
 
     - name: Save JSON data to variable

playbooks/deploy-vms.yml

@@ -32,5 +32,6 @@
     community.general.terraform:
       project_path: "{{ playbook_dir }}/../terraform"
       state: present
+      state_file: "{{ tf_state_file }}"
       variables:
         external_network: "{{ _external_network | to_json }}"

playbooks/vars/main.yml

@@ -66,3 +66,4 @@ default_container_tech: lxc
 osa_enable_infra: True
 osa_enable_compute: True
 osa_enable_ceph: False
+tf_state_file: "{{ lookup('env', 'MNAIO_TF_STATE_FILE') }}"

requirements.txt

@@ -0,0 +1,10 @@
+# The order of packages is significant, because pip processes them in the order
+# of appearance. Changing the order has an impact on the overall integration
+# process, which may cause wedges in the gate later.
+
+pyasn1!=0.2.3,>=0.1.8 # BSD
+pyOpenSSL>=17.1.0 # Apache-2.0
+netaddr>=0.7.18 # BSD
+PrettyTable<0.8,>=0.7.1 # BSD
+python-memcached>=1.56 # PSF
+PyYAML>=3.12 # MIT

scripts/bootstrap.sh

@@ -0,0 +1,151 @@
+#!/usr/bin/env bash
+# Copyright 2023, Rackspace US, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+## Shell Opts ----------------------------------------------------------------
+set -e -u -x
+
+## Vars ----------------------------------------------------------------------
+export HTTP_PROXY=${HTTP_PROXY:-""}
+export HTTPS_PROXY=${HTTPS_PROXY:-""}
+# The Ansible version used for testing
+export ANSIBLE_PACKAGE=${ANSIBLE_PACKAGE:-"ansible-base==2.10.5"}
+export ANSIBLE_ROLE_FILE=${ANSIBLE_ROLE_FILE:-"ansible-role-requirements.yml"}
+export ANSIBLE_COLLECTION_FILE=${ANSIBLE_COLLECTION_FILE:-"ansible-collection-requirements.yml"}
+export USER_ROLE_FILE=${USER_ROLE_FILE:-"user-role-requirements.yml"}
+export USER_COLLECTION_FILE=${USER_COLLECTION_FILE:-"user-collection-requirements.yml"}
+export SSH_DIR=${SSH_DIR:-"/root/.ssh"}
+export DEBIAN_FRONTEND=${DEBIAN_FRONTEND:-"noninteractive"}
+
+# Use pip opts to add options to the pip install command.
+# This can be used to tell it which index to use, etc.
+export PIP_OPTS=${PIP_OPTS:-""}
+
+# This script should be executed from the root directory of the cloned repo
+cd "$(dirname "${0}")/.."
+
+## Functions -----------------------------------------------------------------
+info_block "Checking for required libraries." 2> /dev/null ||
+    source scripts/library.sh
+
+# Create the ssh dir if needed
+ssh_key_create
+
+# Determine the distribution which the host is running on
+determine_distro
+
+# Install the base packages
+case ${DISTRO_ID} in
+    centos|rhel)
+        VERS=${DISTRO_VERSION%%.*}
+        dnf install https://dl.fedoraproject.org/pub/epel/epel-release-latest-${VERS}.noarch.rpm && dnf update
+
+        dnf -y install \
+          git curl autoconf gcc gcc-c++ nc \
+          python3 python3-devel libselinux-python3 \
+          openssl-devel libffi-devel \
+          python3-virtualenv rsync snapd
+
+
+        systemctl enable --now snapd.socket
+        if [ ! -e /snap ]; then
+	  ln -s /var/lib/snapd/snap /snap
+        fi
+
+	snap install --classic terraform
+        ;;
+    ubuntu|debian)
+
+        apt-get update
+        DEBIAN_FRONTEND=noninteractive apt-get -y install \
+          git-core curl gcc netcat \
+          python3 python3-dev \
+          libssl-dev libffi-dev \
+          python3-apt virtualenv \
+          python3-minimal snap
+
+	snap install --classic terraform
+        ;;
+esac
+
+# Ensure that our shell knows about the new virtualenv
+hash -r virtualenv
+
+# Ensure we use the HTTPS/HTTP proxy with pip if it is specified
+if [ -n "$HTTPS_PROXY" ]; then
+  PIP_OPTS+="--proxy $HTTPS_PROXY"
+
+elif [ -n "$HTTP_PROXY" ]; then
+  PIP_OPTS+="--proxy $HTTP_PROXY"
+fi
+
+PYTHON_EXEC_PATH="${PYTHON_EXEC_PATH:-$(which python3 || which python2 || which python)}"
+PYTHON_VERSION="$($PYTHON_EXEC_PATH -c 'import sys; print(".".join(map(str, sys.version_info[:3])))')"
+
+# Use https when Python with native SNI support is available
+UPPER_CONSTRAINTS_PROTO=$([ "$PYTHON_VERSION" == $(echo -e "$PYTHON_VERSION\n2.7.9" | sort -V | tail -1) ] && echo "https" || echo "http")
+
+# Set the location of the constraints to use for all pip installations
+export UPPER_CONSTRAINTS_FILE=${UPPER_CONSTRAINTS_FILE:-"$UPPER_CONSTRAINTS_PROTO://opendev.org/openstack/requirements/raw/commit/485df374e5b52e1c7f3eed243b1518017f0c3f93/upper-constraints.txt"}
+
+
+## Main ----------------------------------------------------------------------
+info_block "Bootstrapping System with Ansible"
+
+# Store the clone repo root location
+export MNAIO_CLONE_DIR="$(pwd)"
+
+if [[ -z "${SKIP_MNAIO_RUNTIME_VENV_BUILD+defined}" ]]; then
+    build_ansible_runtime_venv
+fi
+
+
+# Update dependent roles
+if [ -f "${ANSIBLE_COLLECTION_FILE}" ] && [[ -z "${SKIP_MNAIO_ROLE_CLONE+defined}" ]]; then
+    #export ANSIBLE_LIBRARY="${MNAIO_CLONE_DIR}/playbooks/library"
+    export ANSIBLE_LOOKUP_PLUGINS="/dev/null"
+    export ANSIBLE_FILTER_PLUGINS="/dev/null"
+    export ANSIBLE_ACTION_PLUGINS="/dev/null"
+    export ANSIBLE_CALLBACK_PLUGINS="/dev/null"
+    export ANSIBLE_CALLBACK_WHITELIST="/dev/null"
+    export ANSIBLE_TEST_PLUGINS="/dev/null"
+    export ANSIBLE_VARS_PLUGINS="/dev/null"
+    export ANSIBLE_STRATEGY_PLUGINS="/dev/null"
+    export ANSIBLE_CONFIG="none-ansible.cfg"
+    export ANSIBLE_COLLECTIONS_PATH="/opt/ansible-mnaio2/"
+
+    pushd scripts
+      /opt/ansible-mnaio2/bin/ansible-playbook get-ansible-collection-requirements.yml \
+        -e collection_file="${ANSIBLE_COLLECTION_FILE}" -e user_collection_file="${USER_COLLECTION_FILE}"
+
+      #/opt/ansible-mnaio2/bin/ansible-playbook get-ansible-role-requirements.yml \
+      #  -e role_file="${ANSIBLE_ROLE_FILE}" -e user_role_file="${USER_ROLE_FILE}"
+    popd
+
+    unset ANSIBLE_LIBRARY
+    unset ANSIBLE_LOOKUP_PLUGINS
+    unset ANSIBLE_FILTER_PLUGINS
+    unset ANSIBLE_ACTION_PLUGINS
+    unset ANSIBLE_CALLBACK_PLUGINS
+    unset ANSIBLE_CALLBACK_WHITELIST
+    unset ANSIBLE_TEST_PLUGINS
+    unset ANSIBLE_VARS_PLUGINS
+    unset ANSIBLE_STRATEGY_PLUGINS
+    unset ANSIBLE_CONFIG
+    unset ANSIBLE_COLLECTIONS_PATH
+fi
+
+echo "System is bootstrapped and ready for use."
+

scripts/get-ansible-collection-requirements.yml

@@ -0,0 +1,80 @@
+---
+# Copyright 2020 BBC R&D.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+- name: Clone the role ansible-role-requirements
+  hosts: localhost
+  connection: local
+  user: root
+  gather_facts: false
+  tasks:
+    - name: Ensure the default collections directory exists
+      file:
+        path: "{{ collection_path_default }}"
+        state: directory
+
+    - name: Read the list of user collections
+      set_fact:
+        user_collection_names: "{{ user_collections.collections | default([]) | map(attribute='name') | list }}"
+
+    - name: Generate a list of required collections excluding user overridden collections
+      set_fact:
+         galaxy_collections_list : "{{ galaxy_collections_list + [ item ] }}"
+      when:
+        - item.name not in user_collection_names
+      with_items: "{{ required_collections.collections }}"
+
+    - name: Append user collections to filtered required collections
+      set_fact:
+        galaxy_collections_list: "{{ galaxy_collections_list + [ item ] }}"
+      with_items: "{{ user_collections.collections }}"
+      when:
+        - user_collections.collections is defined
+
+    - name: Create temporary file for galaxy collection requirements
+      tempfile:
+      register: collection_requirements_tmpfile
+
+    - name: Copy content into galaxy collection requirements temporary file
+      copy:
+        content: "{{ galaxy_collections | to_nice_yaml }}"
+        dest: "{{ collection_requirements_tmpfile.path }}"
+
+    - name: Install collection requirements with ansible galaxy
+      command: >
+        /opt/ansible-runtime/bin/ansible-galaxy collection install --force
+        -r "{{ collection_requirements_tmpfile.path }}"
+        -p "{{ collection_path_default }}"
+      register: collection_install
+      until: collection_install is success
+      retries: 5
+      delay: 2
+
+    - name: Show collection install output
+      debug: msg="{{ collection_install.stdout.split('\n') }}"
+
+    - name: Clean up temporary file
+      file:
+        path: "{{ collection_requirements_tmpfile.path }}"
+        state: absent
+
+  vars:
+    galaxy_collections_list: []
+    galaxy_collections:
+      collections: "{{ galaxy_collections_list }}"
+    collections_file: "{{ playbook_dir }}/../ansible-collection-requirements.yml"
+    required_collections: "{{ lookup('file', collections_file) | from_yaml }}"
+    collection_path_default: '/opt/ansible-mnaio2/'
+    user_collections: "{{ lookup('file', user_collections_path, errors='ignore')|default([], true) | from_yaml }}"
+    user_collections_path: "{{ lookup('env','MNAIO_CONFIG_DIR') | default('/opt/mnaiov2', true) ~ '/' ~ (user_collection_file|default('')) }}"