diff --git a/.github/workflows/create-devnet.yml b/.github/workflows/create-devnet.yml new file mode 100644 index 00000000..c1475573 --- /dev/null +++ b/.github/workflows/create-devnet.yml @@ -0,0 +1,307 @@ +name: Create Devnet + +on: + workflow_dispatch: + inputs: + devnet_name: + description: "Devnet name (without 'devnet-' prefix, e.g. 'mytest' creates 'devnet-mytest')" + required: true + type: string + platform_version: + description: "Platform/dashmate version to deploy (e.g. 2.0.0-rc.16)" + required: true + type: string + default: "2.0.0-rc.16" + # Advanced options - sane defaults, only change if you know what you're doing + hp_masternodes_arm_count: + description: "Advanced: Number of ARM HP masternodes" + required: false + type: string + default: "11" + hp_masternodes_amd_count: + description: "Advanced: Number of AMD HP masternodes" + required: false + type: string + default: "0" + masternodes_arm_count: + description: "Advanced: Number of ARM regular masternodes" + required: false + type: string + default: "0" + masternodes_amd_count: + description: "Advanced: Number of AMD regular masternodes" + required: false + type: string + default: "0" + seed_count: + description: "Advanced: Number of seed nodes" + required: false + type: string + default: "1" + core_version: + description: "Advanced: Core (dashd) image version (leave empty for default)" + required: false + type: string + default: "" + hpmn_disk_size: + description: "Advanced: HP masternode disk size in GB" + required: false + type: string + default: "30" + +jobs: + create: + name: Create Devnet + runs-on: ubuntu-latest + timeout-minutes: 120 + concurrency: + group: "devnet-${{ github.event.inputs.devnet_name }}" + cancel-in-progress: false + + env: + NETWORK_NAME: "devnet-${{ github.event.inputs.devnet_name }}" + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + AWS_REGION: ${{ secrets.AWS_REGION }} + TERRAFORM_S3_BUCKET: ${{ secrets.TERRAFORM_S3_BUCKET }} + TERRAFORM_S3_KEY: ${{ secrets.TERRAFORM_S3_KEY }} + TERRAFORM_DYNAMODB_TABLE: ${{ secrets.TERRAFORM_DYNAMODB_TABLE }} + ANSIBLE_HOST_KEY_CHECKING: "false" + + steps: + - name: Validate devnet name + env: + NAME: ${{ github.event.inputs.devnet_name }} + run: | + if [[ -z "$NAME" ]]; then + echo "Error: devnet_name is required" + exit 1 + fi + if [[ "$NAME" =~ ^devnet- ]]; then + echo "Error: Do not include 'devnet-' prefix. Just provide the name (e.g. 'mytest')" + exit 1 + fi + if [[ ! "$NAME" =~ ^[a-z0-9][a-z0-9-]*$ ]]; then + echo "Error: devnet_name must be lowercase alphanumeric with optional hyphens" + exit 1 + fi + echo "Will create: devnet-$NAME" + + - name: Checkout dash-network-deploy + uses: actions/checkout@v4 + + - name: Set up Node.js + uses: actions/setup-node@v4 + with: + node-version: '20' + + - name: Install Node.js dependencies + run: npm ci + + - name: Set up Terraform + uses: hashicorp/setup-terraform@v3 + with: + terraform_version: "1.12.1" + terraform_wrapper: false + + - name: Install system dependencies + run: | + sudo apt-get update + sudo apt-get install -y python3-pip python3-netaddr sshpass jq + + - name: Install Ansible + run: | + python3 -m pip install --upgrade pip + python3 -m pip install ansible + + - name: Install Ansible roles + run: | + ansible-galaxy install -r ansible/requirements.yml + mkdir -p ~/.ansible/roles + cp -r ansible/roles/* ~/.ansible/roles/ + + - name: Set up SSH keys + env: + DEPLOY_SERVER_KEY: ${{ secrets.DEPLOY_SERVER_KEY }} + EVO_APP_DEPLOY_KEY: ${{ secrets.EVO_APP_DEPLOY_KEY }} + run: | + mkdir -p ~/.ssh + + # Server SSH key for connecting to nodes + printf '%s\n' "$DEPLOY_SERVER_KEY" > ~/.ssh/id_rsa + chmod 600 ~/.ssh/id_rsa + + # Derive public key from private key + ssh-keygen -y -f ~/.ssh/id_rsa > ~/.ssh/id_rsa.pub + chmod 644 ~/.ssh/id_rsa.pub + + # GitHub deploy key for cloning configs repo + printf '%s\n' "$EVO_APP_DEPLOY_KEY" > ~/.ssh/id_ed25519 + chmod 600 ~/.ssh/id_ed25519 + + # SSH config + cat > ~/.ssh/config << 'EOL' + Host github.com + IdentityFile ~/.ssh/id_ed25519 + StrictHostKeyChecking no + + Host * + IdentityFile ~/.ssh/id_rsa + User ubuntu + StrictHostKeyChecking no + UserKnownHostsFile=/dev/null + EOL + + chmod 600 ~/.ssh/config + + - name: Create networks/.env + run: | + mkdir -p networks + cat > networks/.env << EOF + PRIVATE_KEY_PATH=$HOME/.ssh/id_rsa + PUBLIC_KEY_PATH=$HOME/.ssh/id_rsa.pub + AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID + AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY + AWS_REGION=$AWS_REGION + TERRAFORM_S3_BUCKET=$TERRAFORM_S3_BUCKET + TERRAFORM_S3_KEY=$TERRAFORM_S3_KEY + TERRAFORM_DYNAMODB_TABLE=$TERRAFORM_DYNAMODB_TABLE + EOF + + - name: Generate network configs + env: + MN_AMD: ${{ github.event.inputs.masternodes_amd_count }} + MN_ARM: ${{ github.event.inputs.masternodes_arm_count }} + HP_AMD: ${{ github.event.inputs.hp_masternodes_amd_count }} + HP_ARM: ${{ github.event.inputs.hp_masternodes_arm_count }} + SEED_COUNT: ${{ github.event.inputs.seed_count }} + run: | + # Validate all counts are numeric + for var in MN_AMD MN_ARM HP_AMD HP_ARM SEED_COUNT; do + val="${!var}" + if [[ ! "$val" =~ ^[0-9]+$ ]]; then + echo "Error: $var must be a number, got '$val'" + exit 1 + fi + done + + echo "Generating configs for $NETWORK_NAME..." + chmod +x ./bin/generate + ./bin/generate "$NETWORK_NAME" \ + "$MN_AMD" "$MN_ARM" "$HP_AMD" "$HP_ARM" \ + -s="$SEED_COUNT" + + echo "Generated config files:" + ls -la networks/devnet-* + + - name: Update platform version in config + env: + VERSION: ${{ github.event.inputs.platform_version }} + CORE_VERSION: ${{ github.event.inputs.core_version }} + run: | + CONFIG_FILE="networks/$NETWORK_NAME.yml" + + # Escape sed-special characters in version strings + SAFE_VERSION=$(printf '%s' "$VERSION" | sed 's/[&\\/]/\\&/g') + SAFE_CORE_VERSION=$(printf '%s' "$CORE_VERSION" | sed 's/[&\\/]/\\&/g') + + echo "Setting platform version to $VERSION..." + + # Update dashmate version + sed -i "s/dashmate_version: .*/dashmate_version: $SAFE_VERSION/" "$CONFIG_FILE" + + # Update platform service images + sed -i "s|drive_image: dashpay/drive:.*|drive_image: dashpay/drive:$SAFE_VERSION|" "$CONFIG_FILE" + sed -i "s|dapi_image: dashpay/dapi:.*|dapi_image: dashpay/dapi:$SAFE_VERSION|" "$CONFIG_FILE" + + # Add rs_dapi_image (not in generated config, but group_vars/all defines it + # without a tag, so we must explicitly set it to get the right version) + if ! grep -q "rs_dapi_image:" "$CONFIG_FILE"; then + echo "rs_dapi_image: dashpay/rs-dapi:$VERSION" >> "$CONFIG_FILE" + else + sed -i "s|rs_dapi_image: dashpay/rs-dapi:.*|rs_dapi_image: dashpay/rs-dapi:$SAFE_VERSION|" "$CONFIG_FILE" + fi + + # Update core version if specified + if [[ -n "$CORE_VERSION" ]]; then + echo "Setting core version to $CORE_VERSION..." + sed -i "s|dashd_image: dashpay/dashd:.*|dashd_image: dashpay/dashd:$SAFE_CORE_VERSION|" "$CONFIG_FILE" + fi + + echo "Updated config:" + grep -E "(dashmate_version|drive_image|dapi_image|rs_dapi_image|dashd_image)" "$CONFIG_FILE" + + - name: Update terraform config + env: + DISK_SIZE: ${{ github.event.inputs.hpmn_disk_size }} + run: | + TFVARS_FILE="networks/$NETWORK_NAME.tfvars" + + # Read current value from file (empty if not set) + CURRENT_SIZE=$(grep -oP 'hpmn_node_disk_size\s*=\s*\K[0-9]+' "$TFVARS_FILE" 2>/dev/null || echo "") + + if [[ -n "$DISK_SIZE" && "$DISK_SIZE" != "$CURRENT_SIZE" ]]; then + if [[ ! "$DISK_SIZE" =~ ^[0-9]+$ ]]; then + echo "Error: hpmn_disk_size must be a number, got '$DISK_SIZE'" + exit 1 + fi + echo "Setting HP masternode disk size to ${DISK_SIZE}GB..." + if [[ -n "$CURRENT_SIZE" ]]; then + sed -i "s/hpmn_node_disk_size = .*/hpmn_node_disk_size = $DISK_SIZE/" "$TFVARS_FILE" + else + echo "hpmn_node_disk_size = $DISK_SIZE" >> "$TFVARS_FILE" + fi + fi + + echo "Terraform config:" + cat "$TFVARS_FILE" + + - name: Deploy devnet (Terraform + Ansible) + run: | + echo "============================================" + echo "Deploying $NETWORK_NAME" + echo "============================================" + + chmod +x ./bin/deploy + # GitHub Actions checks out a detached HEAD; bypass branch safety check. + ./bin/deploy -f "$NETWORK_NAME" + + - name: Push configs to dash-network-configs + run: | + # Clone the configs repo to a temp directory + git clone git@github.com:dashpay/dash-network-configs.git /tmp/dash-network-configs + + # Copy generated config files + cp "networks/$NETWORK_NAME.yml" /tmp/dash-network-configs/ + cp "networks/$NETWORK_NAME.tfvars" /tmp/dash-network-configs/ + cp "networks/$NETWORK_NAME.inventory" /tmp/dash-network-configs/ + + # Commit and push + cd /tmp/dash-network-configs + git config user.name "GitHub Actions" + git config user.email "actions@github.com" + git add . + git commit -m "Add configs for $NETWORK_NAME" || echo "No changes to commit" + git push + + echo "Configs pushed to dash-network-configs repo" + + - name: Verify platform services + if: success() + run: | + echo "Verifying platform services on HP masternodes..." + ansible hp_masternodes \ + -i "networks/$NETWORK_NAME.inventory" \ + --private-key="$HOME/.ssh/id_rsa" \ + -b -m shell \ + -a 'sudo -u dashmate dashmate status services --format=json | jq -r ".[] | select(.service != \"core\") | \"\(.service): \(.status)\""' + + - name: Print summary + if: always() + run: | + echo "============================================" + echo "Devnet: $NETWORK_NAME" + echo "============================================" + echo "" + echo "To update this devnet later, use the 'Platform Version Deployment' workflow" + echo "To destroy this devnet, use the 'Destroy Devnet' workflow" diff --git a/.github/workflows/destroy-devnet.yml b/.github/workflows/destroy-devnet.yml new file mode 100644 index 00000000..a359bafa --- /dev/null +++ b/.github/workflows/destroy-devnet.yml @@ -0,0 +1,221 @@ +name: Destroy Devnet + +on: + workflow_dispatch: + inputs: + devnet_name: + description: "Devnet name (without 'devnet-' prefix, e.g. 'mytest' destroys 'devnet-mytest')" + required: true + type: string + destroy_target: + description: "What to destroy" + required: true + type: choice + default: "all" + options: + - all + - platform + - network + +jobs: + destroy: + name: Destroy Devnet + runs-on: ubuntu-latest + timeout-minutes: 60 + concurrency: + group: "devnet-${{ github.event.inputs.devnet_name }}" + cancel-in-progress: false + + env: + NETWORK_NAME: "devnet-${{ github.event.inputs.devnet_name }}" + DESTROY_TARGET: ${{ github.event.inputs.destroy_target }} + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + AWS_REGION: ${{ secrets.AWS_REGION }} + TERRAFORM_S3_BUCKET: ${{ secrets.TERRAFORM_S3_BUCKET }} + TERRAFORM_S3_KEY: ${{ secrets.TERRAFORM_S3_KEY }} + TERRAFORM_DYNAMODB_TABLE: ${{ secrets.TERRAFORM_DYNAMODB_TABLE }} + ANSIBLE_HOST_KEY_CHECKING: "false" + + steps: + - name: Validate devnet name + env: + NAME: ${{ github.event.inputs.devnet_name }} + run: | + if [[ -z "$NAME" ]]; then + echo "Error: devnet_name is required" + exit 1 + fi + if [[ "$NAME" =~ ^devnet- ]]; then + echo "Error: Do not include 'devnet-' prefix. Just provide the name (e.g. 'mytest')" + exit 1 + fi + echo "Will destroy: devnet-$NAME (target: $DESTROY_TARGET)" + + - name: Checkout dash-network-deploy + uses: actions/checkout@v4 + + - name: Set up Node.js + uses: actions/setup-node@v4 + with: + node-version: '20' + + - name: Install Node.js dependencies + run: npm ci + + - name: Set up Terraform + uses: hashicorp/setup-terraform@v3 + with: + terraform_version: "1.12.1" + terraform_wrapper: false + + - name: Install system dependencies + run: | + sudo apt-get update + sudo apt-get install -y python3-pip python3-netaddr sshpass jq + + - name: Install Ansible + run: | + python3 -m pip install --upgrade pip + python3 -m pip install ansible + + - name: Install Ansible roles + run: | + ansible-galaxy install -r ansible/requirements.yml + mkdir -p ~/.ansible/roles + cp -r ansible/roles/* ~/.ansible/roles/ + + - name: Set up SSH keys + env: + DEPLOY_SERVER_KEY: ${{ secrets.DEPLOY_SERVER_KEY }} + EVO_APP_DEPLOY_KEY: ${{ secrets.EVO_APP_DEPLOY_KEY }} + run: | + mkdir -p ~/.ssh + + # Server SSH key for connecting to nodes + printf '%s\n' "$DEPLOY_SERVER_KEY" > ~/.ssh/id_rsa + chmod 600 ~/.ssh/id_rsa + + # Derive public key from private key + ssh-keygen -y -f ~/.ssh/id_rsa > ~/.ssh/id_rsa.pub + chmod 644 ~/.ssh/id_rsa.pub + + # GitHub deploy key for cloning configs repo + printf '%s\n' "$EVO_APP_DEPLOY_KEY" > ~/.ssh/id_ed25519 + chmod 600 ~/.ssh/id_ed25519 + + # SSH config + cat > ~/.ssh/config << 'EOL' + Host github.com + IdentityFile ~/.ssh/id_ed25519 + StrictHostKeyChecking no + + Host * + IdentityFile ~/.ssh/id_rsa + User ubuntu + StrictHostKeyChecking no + UserKnownHostsFile=/dev/null + EOL + + chmod 600 ~/.ssh/config + + - name: Create networks/.env + run: | + mkdir -p networks + cat > networks/.env << EOF + PRIVATE_KEY_PATH=$HOME/.ssh/id_rsa + PUBLIC_KEY_PATH=$HOME/.ssh/id_rsa.pub + AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID + AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY + AWS_REGION=$AWS_REGION + TERRAFORM_S3_BUCKET=$TERRAFORM_S3_BUCKET + TERRAFORM_S3_KEY=$TERRAFORM_S3_KEY + TERRAFORM_DYNAMODB_TABLE=$TERRAFORM_DYNAMODB_TABLE + EOF + + - name: Clone network configs + run: | + rm -rf networks/.git + git clone git@github.com:dashpay/dash-network-configs.git /tmp/dash-network-configs + + # Copy config files for this devnet + cp /tmp/dash-network-configs/$NETWORK_NAME.yml networks/ 2>/dev/null || true + cp /tmp/dash-network-configs/$NETWORK_NAME.tfvars networks/ 2>/dev/null || true + cp /tmp/dash-network-configs/$NETWORK_NAME.inventory networks/ 2>/dev/null || true + + - name: Validate config files exist + run: | + MISSING=() + for ext in yml tfvars inventory; do + if [[ ! -f "networks/$NETWORK_NAME.$ext" ]]; then + MISSING+=("networks/$NETWORK_NAME.$ext") + fi + done + + if [[ ${#MISSING[@]} -gt 0 ]]; then + echo "Error: Missing config file(s):" + for f in "${MISSING[@]}"; do + echo " - $f" + done + echo "" + echo "Available configs in dash-network-configs:" + ls /tmp/dash-network-configs/*.yml 2>/dev/null || echo " (none)" + exit 1 + fi + + echo "Found all config files for $NETWORK_NAME" + ls -la networks/$NETWORK_NAME.* + + - name: Print destruction plan + run: | + echo "============================================" + echo "WARNING: Destroying $NETWORK_NAME" + echo "Target: $DESTROY_TARGET" + echo "============================================" + echo "" + case "$DESTROY_TARGET" in + all) + echo "This will DESTROY ALL INFRASTRUCTURE (EC2 instances, VPCs, etc.)" + echo "and remove configs from the dash-network-configs repo." + ;; + platform) + echo "This will destroy platform services on HP masternodes." + echo "Infrastructure will be kept." + ;; + network) + echo "This will destroy all services and configs on nodes." + echo "Infrastructure will be kept." + ;; + esac + echo "" + + - name: Destroy devnet + run: | + chmod +x ./bin/destroy + ./bin/destroy "$NETWORK_NAME" -t="$DESTROY_TARGET" + + - name: Remove configs from dash-network-configs + if: github.event.inputs.destroy_target == 'all' + run: | + cd /tmp/dash-network-configs + git config user.name "GitHub Actions" + git config user.email "actions@github.com" + + # Remove config files for this devnet + git rm "$NETWORK_NAME.yml" 2>/dev/null || true + git rm "$NETWORK_NAME.tfvars" 2>/dev/null || true + git rm "$NETWORK_NAME.inventory" 2>/dev/null || true + + git commit -m "Remove configs for $NETWORK_NAME (destroyed)" || echo "No changes to commit" + git push + + echo "Configs removed from dash-network-configs repo" + + - name: Print summary + if: always() + run: | + echo "============================================" + echo "Devnet: $NETWORK_NAME" + echo "Target: $DESTROY_TARGET" + echo "Status: Destruction complete" + echo "============================================" diff --git a/.github/workflows/platform-deploy.yml b/.github/workflows/platform-deploy.yml index 41e8a822..6ec85126 100644 --- a/.github/workflows/platform-deploy.yml +++ b/.github/workflows/platform-deploy.yml @@ -109,9 +109,18 @@ jobs: # Update platform service image versions sed -i "s/drive_image: dashpay\/drive:[^ ]*/drive_image: dashpay\/drive:${{ github.event.inputs.platform_version }}/" networks/${{ github.event.inputs.network }}.yml sed -i "s/dapi_image: dashpay\/dapi:[^ ]*/dapi_image: dashpay\/dapi:${{ github.event.inputs.platform_version }}/" networks/${{ github.event.inputs.network }}.yml - + + # Update rs_dapi_image (group_vars/all defines it without a tag, so we must + # explicitly set it to get the right version instead of :latest) + CONFIG_FILE="networks/${{ github.event.inputs.network }}.yml" + if grep -q "rs_dapi_image:" "$CONFIG_FILE"; then + sed -i "s|rs_dapi_image: dashpay/rs-dapi:[^ ]*|rs_dapi_image: dashpay/rs-dapi:${{ github.event.inputs.platform_version }}|" "$CONFIG_FILE" + else + echo "rs_dapi_image: dashpay/rs-dapi:${{ github.event.inputs.platform_version }}" >> "$CONFIG_FILE" + fi + echo "Updated network config:" - grep -E "(dashmate_version|drive_image|dapi_image)" networks/${{ github.event.inputs.network }}.yml + grep -E "(dashmate_version|drive_image|dapi_image|rs_dapi_image)" networks/${{ github.event.inputs.network }}.yml # Run platform deployment - name: Run Platform Deployment