From b30bc2d8be69237145a38871ab888583054e1761 Mon Sep 17 00:00:00 2001 From: Mangat Toor Date: Mon, 2 Jun 2025 18:09:15 -0700 Subject: [PATCH 1/5] ci: setup terraform script for provisioning dev environment --- terraform/dev/main.tf | 118 +++++++++++++++++++++++++++++++++++++ terraform/dev/variables.tf | 0 2 files changed, 118 insertions(+) create mode 100644 terraform/dev/main.tf create mode 100644 terraform/dev/variables.tf diff --git a/terraform/dev/main.tf b/terraform/dev/main.tf new file mode 100644 index 0000000..21481de --- /dev/null +++ b/terraform/dev/main.tf @@ -0,0 +1,118 @@ +terraform { + backend "s3" { + bucket = "" + key = """ + region = "" + } +} + +provider "aws" { + region = "us-east-1" +} + +# Generate a new SSH key pair +resource "tls_private_key" "ec2_key" { + algorithm = "RSA" + rsa_bits = 4096 +} + +# Create AWS key pair from generated key +resource "aws_key_pair" "deployer" { + key_name = "generated-key" + public_key = tls_private_key.ec2_key.public_key_openssh +} + +# Security Group for RDS +resource "aws_security_group" "db_sg" { + name = "db-sg" + description = "Security group for RDS PostgreSQL instance" + + ingress { + from_port = 5432 + to_port = 5432 + protocol = "tcp" + security_groups = [aws_security_group.app_sg.id] + description = "PostgreSQL access from EC2 app server" + } + + egress { + from_port = 0 + to_port = 0 + protocol = "-1" + cidr_blocks = ["0.0.0.0/0"] + description = "Allow all outbound traffic" + } +} + +# RDS DB +resource "aws_db_instance" "mydb" { + identifier = "myapp-db" + engine = "postgres" + instance_class = "db.t3.micro" + allocated_storage = 20 + db_name = var.db_name + username = var.db_user + password = var.db_password + skip_final_snapshot = true + publicly_accessible = false + # Security groups + vpc_security_group_ids = [aws_security_group.db_sg.id] + +} + +# Security Group for EC2 +resource "aws_security_group" "app_sg" { + name = "app-sg" + + ingress { + from_port = 22 + to_port = 22 + protocol = "tcp" + cidr_blocks = ["0.0.0.0/0"] # SSH + } + + egress { + from_port = 0 + to_port = 0 + protocol = "-1" + cidr_blocks = ["0.0.0.0/0"] + } +} + +# EC2 Instance +resource "aws_instance" "app" { + ami = "ami-0f88e80871fd81e91" + instance_type = "t2.micro" + key_name = aws_key_pair.deployer.key_name + vpc_security_group_ids = [aws_security_group.app_sg.id] + + tags = { + Name = "myapp-ec2" + } + + user_data = <<-EOF + #!/bin/bash + yum update -y + yum install -y docker git + service docker start + usermod -a -G docker ec2-user + chkconfig docker on + dnf install postgresql15 -y + sudo curl -L https://github.com/docker/compose/releases/latest/download/docker-compose-$(uname -s)-$(uname -m) -o /usr/local/bin/docker-compose + sudo chmod +x /usr/local/bin/docker-compose + EOF +} + +# Outputs +output "ec2_ip" { + value = aws_instance.app.public_ip +} + +output "rds_endpoint" { + value = aws_db_instance.mydb.address +} + +output "private_key_pem" { + value = tls_private_key.ec2_key.private_key_pem + sensitive = true +} \ No newline at end of file diff --git a/terraform/dev/variables.tf b/terraform/dev/variables.tf new file mode 100644 index 0000000..e69de29 From ee63c05bb943ef463cd98833d194df41e6fe1d28 Mon Sep 17 00:00:00 2001 From: Mangat Toor Date: Mon, 2 Jun 2025 18:12:28 -0700 Subject: [PATCH 2/5] ci: setup github workflow for terraform provisioning and deployment --- .github/workflows/pipeline.yml | 251 +++++++++++++++++++++++++++++++++ 1 file changed, 251 insertions(+) create mode 100644 .github/workflows/pipeline.yml diff --git a/.github/workflows/pipeline.yml b/.github/workflows/pipeline.yml new file mode 100644 index 0000000..4f9a9f8 --- /dev/null +++ b/.github/workflows/pipeline.yml @@ -0,0 +1,251 @@ +name: CI/CD Pipeline + +concurrency: + group: ci-${{ github.workflow }} + cancel-in-progress: true + +on: + workflow_dispatch: + push: + branches: + - dev + +permissions: + contents: read + actions: write + +jobs: + validate-secrets: + runs-on: ubuntu-latest + steps: + - name: Validate required secrets + run: | + REQUIRED_SECRETS=( + "AWS_ACCESS_KEY_ID" + "AWS_SECRET_ACCESS_KEY" + "BUCKET_NAME" + "BUCKET_KEY" + "DB_PASSWORD" + "DJANGO_SECRET_KEY" + "DJANGO_DEBUG" + "AIRBNB_PUBLIC_API_KEY" + "POSTGRES_USER" + "POSTGRES_DB" + "POSTGRES_HOST_PORT" + "CELERY_BROKER_URL" + "CELERY_RESULT_BACKEND" + "GH_TOKEN" + ) + + MISSING=false + + for secret in "${REQUIRED_SECRETS[@]}"; do + if [ -z "${{ secrets[secret] }}" ]; then + echo "❌ Missing secret: $secret" + MISSING=true + fi + done + + if [ "$MISSING" = true ]; then + echo "❌ One or more required secrets are missing. Failing workflow." + exit 1 + else + echo "✅ All required secrets are set." + fi + + # Check if Terraform files have changed + check-changes: + needs: validate-secrets + runs-on: ubuntu-latest + outputs: + terraform-changed: ${{ steps.terraform-changes.outputs.changed }} + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Check for changes in Terraform directory + id: terraform-changes + run: | + echo "Checking for changes in terraform/dev..." + if ! git diff --name-only ${{ github.sha }} | grep -q '^terraform/dev/'; then + echo "Terraform files changed." + echo "changed=true" >> $GITHUB_OUTPUT + else + echo "No changes in terraform/dev." + echo "changed=false" >> $GITHUB_OUTPUT + fi + # Provision infrastructure (only when terraform changes) + provision: + runs-on: ubuntu-latest + needs: check-changes + steps: + - name: Checkout + if: needs.check-changes.outputs.terraform-changed == 'true' + uses: actions/checkout@v4 + + - name: Configure AWS credentials + if: needs.check-changes.outputs.terraform-changed == 'true' + uses: aws-actions/configure-aws-credentials@v3 + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: us-east-1 + + - name: Setup Terraform + if: needs.check-changes.outputs.terraform-changed == 'true' + uses: hashicorp/setup-terraform@v3 + + - name: Terraform Init and Apply + if: needs.check-changes.outputs.terraform-changed == 'true' + env: + TF_VAR_db_password: ${{ secrets.DB_PASSWORD }} + run: | + cd terraform/dev + terraform init -backend-config="bucket=${{secrets.BUCKET_NAME}}" -backend-config="key=${{secrets.BUCKET_KEY}}" -backend-config="region=us-east-1" + terraform apply -auto-approve + + - name: Save Terraform outputs + if: needs.check-changes.outputs.terraform-changed == 'true' + working-directory: terraform/dev + run: | + terraform output -json > tf_outputs.json + cat tf_outputs.json + + - name: Save Private Key + if: needs.check-changes.outputs.terraform-changed == 'true' + working-directory: terraform/dev + run: | + terraform output -raw private_key_pem > private_key.pem + + - name: Upload secrets + if: needs.check-changes.outputs.terraform-changed == 'true' + uses: hkusu/s3-upload-action@v2 + id: upload + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: 'us-east-1' + aws-bucket: ${{ secrets.BUCKET_NAME }} + # TODO: Need to change this variables are dynamic + destination-dir: 'terraform/dev/' + file-path: './terraform/dev/tf_outputs.json' + # TODO: Maybe should be false I think + output-file-url: 'true' + + # Deploy application (always runs, but waits for provision if it ran) + deploy: + runs-on: ubuntu-latest + needs: [check-changes, provision] + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Print start message + run: | + echo "Starting EC2 deployment workflow..." + echo "Terraform changed: ${{ needs.check-changes.outputs.terraform-changed }}" + echo "Provision job result: ${{ needs.provision.result }}" + + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: us-east-1 + + - name: Download from S3 + # TODO: Again path should be dynamic or in a variable + run: | + aws s3 cp s3://${{ secrets.BUCKET_NAME }}/artifacts/terraform/dev/tf_outputs.json ./tf_outputs.json + echo "✅ Downloaded tf_outputs.json" + ls -la tf_outputs.json + + - name: Display tf_outputs.json content + run: cat ./tf_outputs.json + + - name: Parse Terraform outputs + id: tf + run: | + ec2_ip=$(jq -r '.ec2_ip.value' ./tf_outputs.json) + rds_url=$(jq -r '.rds_endpoint.value' ./tf_outputs.json) + echo "Parsed EC2 IP: $ec2_ip" + echo "Parsed RDS URL: $rds_url" + echo "ec2_ip=$ec2_ip" >> $GITHUB_OUTPUT + echo "rds_url=$rds_url" >> $GITHUB_OUTPUT + + - name: Read private key into environment variable + id: read_key + run: | + echo "Reading private key into environment variable..." + if jq -e '.private_key_pem.value' ./tf_outputs.json > /dev/null; then + echo "PRIVATE_KEY<> $GITHUB_ENV + jq -r '.private_key_pem.value' ./tf_outputs.json >> $GITHUB_ENV + echo "EOF" >> $GITHUB_ENV + echo "✅ Private key successfully added to environment variable." + else + echo "❌ Failed to parse private key from tf_outputs.json" + exit 1 + fi + + - name: Deploy to EC2 instance + uses: appleboy/ssh-action@v1.2.0 + with: + host: ${{ steps.tf.outputs.ec2_ip }} + username: ec2-user + key: ${{ env.PRIVATE_KEY }} + # TODO: Move the variables into a env file + script: | + echo "Connected to EC2 instance at ${{ steps.tf.outputs.ec2_ip }}" + git config --global credential.helper store + echo "https://username:${{ secrets.GH_TOKEN }}@github.com" > ~/.git-credentials + cd /home/ec2-user/ + echo "Checking if project directory exists..." + if [ ! -d "airbnb-regulation" ]; then + echo "Cloning project repository..." + git clone https://github.com/CodeForBc/airbnb-regulation + fi + + cd airbnb-regulation + echo "Pulling latest changes from main branch..." + git pull origin main + + echo "Creating .env file with environment variables..." + cat < .env + SECRET_KEY="${{ secrets.DJANGO_SECRET_KEY }}" + DJANGO_DEBUG=${{ secrets.DJANGO_DEBUG }} + AIRBNB_PUBLIC_API_KEY="${{ secrets.AIRBNB_PUBLIC_API_KEY }}" + POSTGRES_PASSWORD=${{ secrets.DB_PASSWORD }} + POSTGRES_USER=${{ secrets.POSTGRES_USER }} + POSTGRES_URL=${{ steps.tf.outputs.rds_url }} + POSTGRES_DB=${{ secrets.POSTGRES_DB }} + POSTGRES_HOST_PORT=${{ secrets.POSTGRES_HOST_PORT }} + CELERY_BROKER_URL=${{ secrets.CELERY_BROKER_URL }} + CELERY_RESULT_BACKEND=${{ secrets.CELERY_RESULT_BACKEND }} + EOF + echo ".env file created." + + if docker ps --format '{{.Names}}' | grep -q '^airbnb_celery$'; then + while true; do + result=$(docker exec airbnb_celery celery -A airbnb_project inspect active 2>&1) + echo "Celery active task output: $result" + + if [[ "$result" == *"empty"* ]]; then + echo "No active Celery tasks detected." + break + fi + + echo "Active Celery tasks found. Waiting..." + sleep 10 + done + else + echo "Celery container 'airbnb_celery' is not running. Skipping task check." + fi + + echo "Stopping Docker containers..." + docker-compose down + + echo "Rebuilding and starting Docker containers..." + docker-compose up -d --build + + echo "Deployment process completed." \ No newline at end of file From 58279cd83a6b0663e313a1c2393fef37ae193841 Mon Sep 17 00:00:00 2001 From: Mangat Toor Date: Sat, 7 Jun 2025 10:39:05 -0700 Subject: [PATCH 3/5] chore: increase retry to give more time for db to come online --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index d17f409..bd06ac9 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -21,7 +21,7 @@ services: test: ["CMD-SHELL", "pg_isready -d $${POSTGRES_DB} -U $${POSTGRES_USER}"] interval: 30s timeout: 10s - retries: 5 + retries: 15 policies: container_name: airbnb_policies From 9e4caaab7fbfab6be226ea320d45011db765a407 Mon Sep 17 00:00:00 2001 From: Mangat Toor Date: Mon, 2 Jun 2025 18:09:15 -0700 Subject: [PATCH 4/5] ci: Implement CI/CD pipeline with Terraform Adds a GitHub Actions workflow to automate the deployment process. - Provisions infrastructure on AWS using Terraform. - Deploys the application to an EC2 instance. - Includes secret validation and a cron job setup script. --- .github/workflows/pipeline.yml | 260 +++++++++++++++++++++++++++++++++ docker-compose.yml | 2 +- scripts/setup_cron.sh | 13 ++ terraform/dev/main.tf | 118 +++++++++++++++ terraform/dev/variables.tf | 7 + 5 files changed, 399 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/pipeline.yml create mode 100644 scripts/setup_cron.sh create mode 100644 terraform/dev/main.tf create mode 100644 terraform/dev/variables.tf diff --git a/.github/workflows/pipeline.yml b/.github/workflows/pipeline.yml new file mode 100644 index 0000000..fe9d2be --- /dev/null +++ b/.github/workflows/pipeline.yml @@ -0,0 +1,260 @@ +name: CI/CD Pipeline + +concurrency: + group: ci-${{ github.workflow }} + cancel-in-progress: true + +on: + workflow_dispatch: + push: + branches: + - dev + +permissions: + contents: read + actions: write + +jobs: + validate-secrets: + runs-on: ubuntu-latest + env: + SECRETS_CONTEXT: ${{ toJson(secrets) }} + steps: + - name: Validate required secrets + run: | + echo "🔍 Validating required secrets..." + + # Required secrets list + REQUIRED_SECRETS=( + "AWS_ACCESS_KEY_ID_DEV" + "AWS_SECRET_ACCESS_KEY_DEV" + "BUCKET_NAME_DEV" + "BUCKET_KEY_DEV" + "DB_PASSWORD_DEV" + "DJANGO_SECRET_KEY_DEV" + "DJANGO_DEBUG_DEV" + "AIRBNB_PUBLIC_API_KEY_DEV" + "POSTGRES_USER_DEV" + "POSTGRES_DB_DEV" + "POSTGRES_HOST_PORT_DEV" + "CELERY_BROKER_URL_DEV" + "CELERY_RESULT_BACKEND_DEV" + ) + + MISSING=false + + for secret in "${REQUIRED_SECRETS[@]}"; do + if ! echo "$SECRETS_CONTEXT" | jq -e --arg key "$secret" 'has($key)' >/dev/null; then + echo "❌ Missing secret: $secret" + MISSING=true + else + echo "✅ Found secret: $secret" + fi + done + + if [ "$MISSING" = true ]; then + echo "❌ One or more required secrets are missing. Failing workflow." + exit 1 + else + echo "✅ All required secrets are set." + fi + # Check if Terraform files have changed + check-changes: + needs: validate-secrets + runs-on: ubuntu-latest + outputs: + terraform-changed: ${{ steps.terraform-changes.outputs.changed }} + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Check for changes in Terraform directory + id: terraform-changes + run: | + echo "Checking for changes in terraform/dev..." + if ! git diff --name-only ${{ github.sha }} | grep -q '^terraform/dev/'; then + echo "Terraform files changed." + echo "changed=true" >> $GITHUB_OUTPUT + else + echo "No changes in terraform/dev." + echo "changed=false" >> $GITHUB_OUTPUT + fi + # Provision infrastructure (only when terraform changes) + provision: + runs-on: ubuntu-latest + needs: check-changes + steps: + - name: Checkout + if: needs.check-changes.outputs.terraform-changed == 'true' + uses: actions/checkout@v4 + + - name: Configure AWS credentials + if: needs.check-changes.outputs.terraform-changed == 'true' + uses: aws-actions/configure-aws-credentials@v3 + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID_DEV }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY_DEV }} + aws-region: us-east-1 + + - name: Setup Terraform + if: needs.check-changes.outputs.terraform-changed == 'true' + uses: hashicorp/setup-terraform@v3 + + - name: Terraform Init and Apply + if: needs.check-changes.outputs.terraform-changed == 'true' + env: + TF_VAR_db_password: ${{ secrets.DB_PASSWORD_DEV }} + run: | + cd terraform/dev + terraform init -backend-config="bucket=${{secrets.BUCKET_NAME_DEV}}" -backend-config="key=${{secrets.BUCKET_KEY_DEV}}" -backend-config="region=us-east-1" + terraform apply -auto-approve \ + -var="db_name_dev=${{ secrets.POSTGRES_DB_DEV }}" \ + -var="db_user_dev=${{ secrets.POSTGRES_USER_DEV }}" \ + -var="db_password_dev=${{ secrets.DB_PASSWORD_DEV }}" + + - name: Save Terraform outputs + if: needs.check-changes.outputs.terraform-changed == 'true' + working-directory: terraform/dev + run: | + terraform output -json > tf_outputs.json + cat tf_outputs.json + + - name: Save Private Key + if: needs.check-changes.outputs.terraform-changed == 'true' + working-directory: terraform/dev + run: | + terraform output -raw private_key_pem > private_key.pem + + - name: Upload secrets + if: needs.check-changes.outputs.terraform-changed == 'true' + uses: hkusu/s3-upload-action@v2 + id: upload + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID_DEV }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY_DEV }} + aws-region: 'us-east-1' + aws-bucket: ${{ secrets.BUCKET_NAME_DEV }} + bucket-root: '/' + # TODO: Need to change this variables are dynamic + destination-dir: '/' + file-path: './terraform/dev/tf_outputs.json' + # TODO: Maybe should be false I think + + # Deploy application (always runs, but waits for provision if it ran) + deploy: + runs-on: ubuntu-latest + needs: [ check-changes, provision ] + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Print start message + run: | + echo "Starting EC2 deployment workflow..." + echo "Terraform changed: ${{ needs.check-changes.outputs.terraform-changed }}" + echo "Provision job result: ${{ needs.provision.result }}" + + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID_DEV }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY_DEV }} + aws-region: us-east-1 + + - name: Download from S3 + # TODO: Again path should be dynamic or in a variable + run: | + aws s3 cp s3://${{ secrets.BUCKET_NAME_DEV}}/tf_outputs.json ./tf_outputs.json + echo "✅ Downloaded tf_outputs.json" + ls -la tf_outputs.json + + - name: Display tf_outputs.json content + run: cat ./tf_outputs.json + + - name: Parse Terraform outputs + id: tf + run: | + ec2_ip=$(jq -r '.ec2_ip.value' ./tf_outputs.json) + rds_url=$(jq -r '.rds_endpoint.value' ./tf_outputs.json) + echo "Parsed EC2 IP: $ec2_ip" + echo "Parsed RDS URL: $rds_url" + echo "ec2_ip=$ec2_ip" >> $GITHUB_OUTPUT + echo "rds_url=$rds_url" >> $GITHUB_OUTPUT + + - name: Read private key into environment variable + id: read_key + run: | + echo "Reading private key into environment variable..." + if jq -e '.private_key_pem.value' ./tf_outputs.json > /dev/null; then + echo "PRIVATE_KEY<> $GITHUB_ENV + jq -r '.private_key_pem.value' ./tf_outputs.json >> $GITHUB_ENV + echo "EOF" >> $GITHUB_ENV + echo "✅ Private key successfully added to environment variable." + else + echo "❌ Failed to parse private key from tf_outputs.json" + exit 1 + fi + + - name: Deploy to EC2 instance + uses: appleboy/ssh-action@v1.2.0 + with: + host: ${{ steps.tf.outputs.ec2_ip }} + username: ec2-user + key: ${{ env.PRIVATE_KEY }} + # TODO: Move the variables into a env file + script: | + echo "Connected to EC2 instance at ${{ steps.tf.outputs.ec2_ip }}" + cd /home/ec2-user/ + echo "Checking if project directory exists..." + if [ ! -d "airbnb-regulation" ]; then + echo "Cloning project repository..." + git clone https://github.com/CodeForBc/airbnb-regulation + fi + + cd airbnb-regulation + echo "Pulling latest changes from main branch..." + git pull origin dev + + echo "Creating .env file with environment variables..." + cat < .env + SECRET_KEY="${{ secrets.DJANGO_SECRET_KEY_DEV }}" + DJANGO_DEBUG=${{ secrets.DJANGO_DEBUG_DEV }} + AIRBNB_PUBLIC_API_KEY="${{ secrets.AIRBNB_PUBLIC_API_KEY_DEV }}" + POSTGRES_PASSWORD=${{ secrets.DB_PASSWORD_DEV }} + POSTGRES_USER=${{ secrets.POSTGRES_USER_DEV }} + POSTGRES_URL=${{ steps.tf.outputs.rds_url}} + POSTGRES_DB=${{ secrets.POSTGRES_DB_DEV }} + POSTGRES_HOST_PORT=${{ secrets.POSTGRES_HOST_PORT_DEV }} + CELERY_BROKER_URL=${{ secrets.CELERY_BROKER_URL_DEV }} + CELERY_RESULT_BACKEND=${{ secrets.CELERY_RESULT_BACKEND_DEV }} + EOF + echo ".env file created." + + if docker ps --format '{{.Names}}' | grep -q '^airbnb_celery$'; then + while true; do + result=$(docker exec airbnb_celery celery -A airbnb_project inspect active 2>&1) + echo "Celery active task output: $result" + + if [[ "$result" == *"empty"* ]]; then + echo "No active Celery tasks detected." + break + fi + + echo "Active Celery tasks found. Waiting..." + sleep 10 + done + else + echo "Celery container 'airbnb_celery' is not running. Skipping task check." + fi + + echo "Stopping Docker containers..." + docker-compose down + + echo "Rebuilding and starting Docker containers..." + docker-compose up -d --build + + echo "Setting up cron jobs..." + chmod +x scripts/setup_cron.sh + ./scripts/setup_cron.sh + echo "✅ Deployment process completed." \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index d17f409..bd06ac9 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -21,7 +21,7 @@ services: test: ["CMD-SHELL", "pg_isready -d $${POSTGRES_DB} -U $${POSTGRES_USER}"] interval: 30s timeout: 10s - retries: 5 + retries: 15 policies: container_name: airbnb_policies diff --git a/scripts/setup_cron.sh b/scripts/setup_cron.sh new file mode 100644 index 0000000..7da5108 --- /dev/null +++ b/scripts/setup_cron.sh @@ -0,0 +1,13 @@ +#!/bin/bash + +CRON_JOB="0 0 * * * curl -s http://localhost:8001/listings/harvest-listings/ > /dev/null 2>&1" + +# Check if the cron job already exists +crontab -l 2>/dev/null | grep -F "$CRON_JOB" >/dev/null + +if [ $? -eq 0 ]; then + echo "Cron job already exists." +else + (crontab -l 2>/dev/null; echo "$CRON_JOB") | crontab - + echo "Cron job added successfully." +fi diff --git a/terraform/dev/main.tf b/terraform/dev/main.tf new file mode 100644 index 0000000..6bb51e5 --- /dev/null +++ b/terraform/dev/main.tf @@ -0,0 +1,118 @@ +terraform { + backend "s3" { + bucket = "" + key = "" + region = "" + } +} + +provider "aws" { + region = "us-east-1" +} + +# Generate a new SSH key pair +resource "tls_private_key" "ec2_key" { + algorithm = "RSA" + rsa_bits = 4096 +} + +# Create AWS key pair from generated key +resource "aws_key_pair" "deployer" { + key_name = "generated-key" + public_key = tls_private_key.ec2_key.public_key_openssh +} + +# Security Group for RDS +resource "aws_security_group" "db_sg" { + name = "db-sg" + description = "Security group for RDS PostgreSQL instance" + + ingress { + from_port = 5432 + to_port = 5432 + protocol = "tcp" + security_groups = [aws_security_group.app_sg.id] + description = "PostgreSQL access from EC2 app server" + } + + egress { + from_port = 0 + to_port = 0 + protocol = "-1" + cidr_blocks = ["0.0.0.0/0"] + description = "Allow all outbound traffic" + } +} + +# RDS DB +resource "aws_db_instance" "mydb" { + identifier = "myapp-db" + engine = "postgres" + instance_class = "db.t3.micro" + allocated_storage = 20 + db_name = var.db_name_dev + username = var.db_user_dev + password = var.db_password_dev + skip_final_snapshot = true + publicly_accessible = false + # Security groups + vpc_security_group_ids = [aws_security_group.db_sg.id] + +} + +# Security Group for EC2 +resource "aws_security_group" "app_sg" { + name = "app-sg" + + ingress { + from_port = 22 + to_port = 22 + protocol = "tcp" + cidr_blocks = ["0.0.0.0/0"] # SSH + } + + egress { + from_port = 0 + to_port = 0 + protocol = "-1" + cidr_blocks = ["0.0.0.0/0"] + } +} + +# EC2 Instance +resource "aws_instance" "app" { + ami = "ami-0f88e80871fd81e91" + instance_type = "t2.micro" + key_name = aws_key_pair.deployer.key_name + vpc_security_group_ids = [aws_security_group.app_sg.id] + + tags = { + Name = "myapp-ec2" + } + + user_data = <<-EOF + #!/bin/bash + yum update -y + yum install -y docker git + service docker start + usermod -a -G docker ec2-user + chkconfig docker on + dnf install postgresql15 -y + sudo curl -L https://github.com/docker/compose/releases/latest/download/docker-compose-$(uname -s)-$(uname -m) -o /usr/local/bin/docker-compose + sudo chmod +x /usr/local/bin/docker-compose + EOF +} + +# Outputs +output "ec2_ip" { + value = aws_instance.app.public_ip +} + +output "rds_endpoint" { + value = aws_db_instance.mydb.address +} + +output "private_key_pem" { + value = tls_private_key.ec2_key.private_key_pem + sensitive = true +} \ No newline at end of file diff --git a/terraform/dev/variables.tf b/terraform/dev/variables.tf new file mode 100644 index 0000000..b5e5f97 --- /dev/null +++ b/terraform/dev/variables.tf @@ -0,0 +1,7 @@ +variable "bucket_name_dev" {default = ""} +variable "bucket_key_dev" {default = ""} +variable "key_name_dev" {default = ""} +variable "public_key_path_dev" {default = ""} +variable "db_name_dev" { default = "" } +variable "db_user_dev" { default = "" } +variable "db_password_dev" {default = ""} From ff570ed7e1eb0d074b36f53612dcc66ba9f386cf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mangat=20Singh=20Toor=20=7C=20=E0=A8=AE=E0=A9=B0=E0=A8=97?= =?UTF-8?q?=E0=A8=A4=20=E0=A8=B8=E0=A8=BF=E0=A9=B0=E0=A8=98=20=E0=A8=A4?= =?UTF-8?q?=E0=A9=82=E0=A8=B0?= Date: Sun, 15 Jun 2025 16:01:49 -0700 Subject: [PATCH 5/5] add prod pipeline terraform and github action --- .github/workflows/pipeline-prod.yml | 292 ++++++++++++++++++++++++++++ terraform/dev/main.tf | 17 +- terraform/prod/main.tf | 133 +++++++++++++ terraform/prod/variables.tf | 7 + 4 files changed, 448 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/pipeline-prod.yml create mode 100644 terraform/prod/main.tf create mode 100644 terraform/prod/variables.tf diff --git a/.github/workflows/pipeline-prod.yml b/.github/workflows/pipeline-prod.yml new file mode 100644 index 0000000..22f18a6 --- /dev/null +++ b/.github/workflows/pipeline-prod.yml @@ -0,0 +1,292 @@ +name: CI/CD Pipeline Prod + +concurrency: + group: ci-${{ github.workflow }} + cancel-in-progress: true + +on: + workflow_dispatch: + push: + branches: + - main + +permissions: + contents: read + actions: write + +jobs: + validate-secrets: + runs-on: ubuntu-latest + env: + SECRETS_CONTEXT: ${{ toJson(secrets) }} + steps: + - name: Validate required secrets + run: | + echo "🔍 Validating required secrets..." + + # Required secrets list (reduced - only AWS and S3 keys needed) + REQUIRED_SECRETS=( + "AWS_ACCESS_KEY_ID_PROD" + "AWS_SECRET_ACCESS_KEY_PROD" + "BUCKET_NAME_PROD" + "BUCKET_KEY_PROD" + "ENV_FILE_S3_KEY_PROD" + ) + + MISSING=false + + for secret in "${REQUIRED_SECRETS[@]}"; do + if ! echo "$SECRETS_CONTEXT" | jq -e --arg key "$secret" 'has($key)' >/dev/null; then + echo "❌ Missing secret: $secret" + MISSING=true + else + echo "✅ Found secret: $secret" + fi + done + + if [ "$MISSING" = true ]; then + echo "❌ One or more required secrets are missing. Failing workflow." + exit 1 + else + echo "✅ All required secrets are set." + fi + + # Check if Terraform files have changed + check-changes: + needs: validate-secrets + runs-on: ubuntu-latest + outputs: + terraform-changed: ${{ steps.terraform-changes.outputs.changed }} + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Check for changes in Terraform directory + id: terraform-changes + run: | + echo "Checking for changes in terraform/prod..." + if git diff --name-only ${{ github.sha }} | grep -q '^terraform/prod/'; then + echo "Terraform files changed." + echo "changed=true" >> $GITHUB_OUTPUT + else + echo "No changes in terraform/prod." + echo "changed=true" >> $GITHUB_OUTPUT + fi + + # Provision infrastructure (only when terraform changes) + provision: + runs-on: ubuntu-latest + needs: check-changes + steps: + - name: Checkout + if: needs.check-changes.outputs.terraform-changed == 'true' + uses: actions/checkout@v4 + + - name: Configure AWS credentials + if: needs.check-changes.outputs.terraform-changed == 'true' + uses: aws-actions/configure-aws-credentials@v3 + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID_PROD }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY_PROD }} + aws-region: us-east-1 + + - name: Setup Terraform + if: needs.check-changes.outputs.terraform-changed == 'true' + uses: hashicorp/setup-terraform@v3 + + - name: Download environment file from S3 + if: needs.check-changes.outputs.terraform-changed == 'true' + run: | + echo "Downloading environment file from S3..." + aws s3 cp s3://${{ secrets.BUCKET_NAME_PROD }}/${{ secrets.ENV_FILE_S3_KEY_PROD }} ./terraform_env.txt + echo "✅ Downloaded environment file" + + - name: Load environment variables from S3 file + if: needs.check-changes.outputs.terraform-changed == 'true' + run: | + echo "Loading environment variables from S3 file..." + sed -i 's/\r$//' terraform_env.txt + while IFS='=' read -r key value; do + # Skip empty lines and comments + if [[ -n "$key" && ! "$key" =~ ^# ]]; then + echo "Setting $key" + echo "$key=$value" >> $GITHUB_ENV + fi + done < terraform_env.txt + + - name: Terraform Init and Apply + if: needs.check-changes.outputs.terraform-changed == 'true' + run: | + cd terraform/prod + terraform init -backend-config="bucket=${{secrets.BUCKET_NAME_PROD}}" -backend-config="key=${{secrets.BUCKET_KEY_PROD}}" -backend-config="region=us-east-1" + terraform apply -auto-approve \ + -var="db_name_prod=$POSTGRES_DB" \ + -var="db_user_prod=$POSTGRES_USER" \ + -var="db_password_prod=$POSTGRES_PASSWORD" + + - name: Save Terraform outputs + if: needs.check-changes.outputs.terraform-changed == 'true' + working-directory: terraform/prod + run: | + terraform output -json > tf_outputs.json + cat tf_outputs.json + + - name: Save Private Key + if: needs.check-changes.outputs.terraform-changed == 'true' + working-directory: terraform/prod + run: | + terraform output -raw private_key_pem > private_key.pem + + - name: Upload secrets + if: needs.check-changes.outputs.terraform-changed == 'true' + uses: hkusu/s3-upload-action@v2 + id: upload + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID_PROD }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY_PROD }} + aws-region: 'us-east-1' + aws-bucket: ${{ secrets.BUCKET_NAME_PROD }} + bucket-root: '/' + destination-dir: '/' + file-path: './terraform/prod/tf_outputs.json' + + # Deploy application (always runs, but waits for provision if it ran) + deploy: + runs-on: ubuntu-latest + needs: [ check-changes, provision ] + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Print start message + run: | + echo "Starting EC2 deployment workflow..." + echo "Terraform changed: ${{ needs.check-changes.outputs.terraform-changed }}" + echo "Provision job result: ${{ needs.provision.result }}" + + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID_PROD }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY_PROD }} + aws-region: us-east-1 + + - name: Download Terraform outputs from S3 + run: | + aws s3 cp s3://${{ secrets.BUCKET_NAME_PROD}}/tf_outputs.json ./tf_outputs.json + echo "✅ Downloaded tf_outputs.json" + ls -la tf_outputs.json + + - name: Download environment file from S3 + run: | + echo "Downloading environment file from S3..." + aws s3 cp s3://${{ secrets.BUCKET_NAME_PROD }}/${{ secrets.ENV_FILE_S3_KEY_PROD }} ./prod_env.txt + echo "✅ Downloaded environment file" + echo "Environment file contents (masked):" + sed 's/=.*/=***/' ./prod_env.txt + + - name: Display tf_outputs.json content + run: cat ./tf_outputs.json + + - name: Parse Terraform outputs + id: tf + run: | + ec2_ip=$(jq -r '.ec2_ip.value' ./tf_outputs.json) + rds_url=$(jq -r '.rds_endpoint.value' ./tf_outputs.json) + echo "Parsed EC2 IP: $ec2_ip" + echo "Parsed RDS URL: $rds_url" + echo "ec2_ip=$ec2_ip" >> $GITHUB_OUTPUT + echo "rds_url=$rds_url" >> $GITHUB_OUTPUT + + - name: Read private key into environment variable + id: read_key + run: | + echo "Reading private key into environment variable..." + if jq -e '.private_key_pem.value' ./tf_outputs.json > /dev/null; then + echo "PRIVATE_KEY<> $GITHUB_ENV + jq -r '.private_key_pem.value' ./tf_outputs.json >> $GITHUB_ENV + echo "EOF" >> $GITHUB_ENV + echo "✅ Private key successfully added to environment variable." + else + echo "❌ Failed to parse private key from tf_outputs.json" + exit 1 + fi + + - name: Transfer environment file to EC2 + uses: appleboy/scp-action@v0.1.7 + with: + host: ${{ steps.tf.outputs.ec2_ip }} + username: ec2-user + key: ${{ env.PRIVATE_KEY }} + source: "./prod_env.txt" + target: "/home/ec2-user/" + + - name: Deploy to EC2 instance + uses: appleboy/ssh-action@v1.2.0 + with: + host: ${{ steps.tf.outputs.ec2_ip }} + username: ec2-user + key: ${{ env.PRIVATE_KEY }} + script: | + echo "Connected to EC2 instance at ${{ steps.tf.outputs.ec2_ip }}" + cd /home/ec2-user/ + echo "Checking if project directory exists..." + if [ ! -d "airbnb-regulation" ]; then + echo "Cloning project repository..." + git clone https://github.com/CodeForBc/airbnb-regulation + fi + + cd airbnb-regulation + echo "Pulling latest changes from main branch..." + git pull origin main + + echo "Creating .env file from S3 environment file..." + if [ -f "/home/ec2-user/prod_env.txt" ]; then + # Copy the S3 environment file to .env + cp /home/ec2-user/prod_env.txt .env + + # Add dynamic values from Terraform outputs + echo "POSTGRES_URL=${{ steps.tf.outputs.rds_url}}" >> .env + + echo "✅ .env file created successfully from S3 configuration" + echo "Environment variables loaded:" + # Show keys only (not values) for security + grep -E '^[^#]' .env | cut -d'=' -f1 | sort + + # Clean up the temp file + rm -f /home/ec2-user/prod_env.txt + else + echo "❌ Environment file not found at /home/ec2-user/prod_env.txt" + exit 1 + fi + + # Check for running Celery tasks before deployment + if docker ps --format '{{.Names}}' | grep -q '^airbnb_celery$'; then + while true; do + result=$(docker exec airbnb_celery celery -A airbnb_project inspect active 2>&1) + echo "Celery active task output: $result" + + if [[ "$result" == *"empty"* ]]; then + echo "No active Celery tasks detected." + break + fi + + echo "Active Celery tasks found. Waiting..." + sleep 10 + done + else + echo "Celery container 'airbnb_celery' is not running. Skipping task check." + fi + + echo "Stopping Docker containers..." + docker-compose down + + echo "Rebuilding and starting Docker containers..." + docker-compose up -d --build + + echo "Setting up cron jobs..." + chmod +x scripts/setup_cron.sh + ./scripts/setup_cron.sh + + echo "🎉 Deployment completed successfully!" \ No newline at end of file diff --git a/terraform/dev/main.tf b/terraform/dev/main.tf index 6bb51e5..19acefc 100644 --- a/terraform/dev/main.tf +++ b/terraform/dev/main.tf @@ -71,6 +71,19 @@ resource "aws_security_group" "app_sg" { cidr_blocks = ["0.0.0.0/0"] # SSH } + ingress { + from_port = 80 + to_port = 80 + protocol = "tcp" + cidr_blocks = ["0.0.0.0/0"] # HTTP access + } + + ingress { + from_port = 443 + to_port = 443 + protocol = "tcp" + cidr_blocks = ["0.0.0.0/0"] # HTTPS access + } egress { from_port = 0 to_port = 0 @@ -93,7 +106,9 @@ resource "aws_instance" "app" { user_data = <<-EOF #!/bin/bash yum update -y - yum install -y docker git + yum install -y docker git cronie + systemctl enable crond + systemctl start crond service docker start usermod -a -G docker ec2-user chkconfig docker on diff --git a/terraform/prod/main.tf b/terraform/prod/main.tf new file mode 100644 index 0000000..1b95d47 --- /dev/null +++ b/terraform/prod/main.tf @@ -0,0 +1,133 @@ +terraform { + backend "s3" { + bucket = "" + key = "" + region = "" + } +} + +provider "aws" { + region = "us-east-1" +} + +# Generate a new SSH key pair +resource "tls_private_key" "ec2_key" { + algorithm = "RSA" + rsa_bits = 4096 +} + +# Create AWS key pair from generated key +resource "aws_key_pair" "deployer" { + key_name = "generated-key" + public_key = tls_private_key.ec2_key.public_key_openssh +} + +# Security Group for RDS +resource "aws_security_group" "db_sg" { + name = "db-sg" + description = "Security group for RDS PostgreSQL instance" + + ingress { + from_port = 5432 + to_port = 5432 + protocol = "tcp" + security_groups = [aws_security_group.app_sg.id] + description = "PostgreSQL access from EC2 app server" + } + + egress { + from_port = 0 + to_port = 0 + protocol = "-1" + cidr_blocks = ["0.0.0.0/0"] + description = "Allow all outbound traffic" + } +} + +# RDS DB +resource "aws_db_instance" "mydb" { + identifier = "myapp-db" + engine = "postgres" + instance_class = "db.t3.micro" + allocated_storage = 20 + db_name = var.db_name_prod + username = var.db_user_prod + password = var.db_password_prod + skip_final_snapshot = true + publicly_accessible = false + # Security groups + vpc_security_group_ids = [aws_security_group.db_sg.id] + +} + +# Security Group for EC2 +resource "aws_security_group" "app_sg" { + name = "app-sg" + + ingress { + from_port = 22 + to_port = 22 + protocol = "tcp" + cidr_blocks = ["0.0.0.0/0"] # SSH + } + ingress { + from_port = 80 + to_port = 80 + protocol = "tcp" + cidr_blocks = ["0.0.0.0/0"] # HTTP access + } + + ingress { + from_port = 443 + to_port = 443 + protocol = "tcp" + cidr_blocks = ["0.0.0.0/0"] # HTTPS access + } + + egress { + from_port = 0 + to_port = 0 + protocol = "-1" + cidr_blocks = ["0.0.0.0/0"] + } + } + + # EC2 Instance + resource "aws_instance" "app" { + ami = "ami-0f88e80871fd81e91" + instance_type = "t2.micro" + key_name = aws_key_pair.deployer.key_name + vpc_security_group_ids = [aws_security_group.app_sg.id] + + tags = { + Name = "myapp-ec2" + } + + user_data = <<-EOF + #!/bin/bash + yum update -y + yum install -y docker git cronie + systemctl enable crond + systemctl start crond + service docker start + usermod -a -G docker ec2-user + chkconfig docker on + dnf install postgresql15 -y + sudo curl -L https://github.com/docker/compose/releases/latest/download/docker-compose-$(uname -s)-$(uname -m) -o /usr/local/bin/docker-compose + sudo chmod +x /usr/local/bin/docker-compose + EOF + } + + # Outputs + output "ec2_ip" { + value = aws_instance.app.public_ip + } + + output "rds_endpoint" { + value = aws_db_instance.mydb.address + } + + output "private_key_pem" { + value = tls_private_key.ec2_key.private_key_pem + sensitive = true + } \ No newline at end of file diff --git a/terraform/prod/variables.tf b/terraform/prod/variables.tf new file mode 100644 index 0000000..f271f0d --- /dev/null +++ b/terraform/prod/variables.tf @@ -0,0 +1,7 @@ +variable "bucket_name_prod" {default = ""} +variable "bucket_key_prod" {default = ""} +variable "key_name_prod" {default = ""} +variable "public_key_path_prod" {default = ""} +variable "db_name_prod" { default = "" } +variable "db_user_prod" { default = "" } +variable "db_password_prod" {default = ""}