diff --git a/.github/workflows/ansible-deploy.yml b/.github/workflows/ansible-deploy.yml new file mode 100644 index 0000000000..b9609bd9ff --- /dev/null +++ b/.github/workflows/ansible-deploy.yml @@ -0,0 +1,89 @@ +name: Ansible Deployment + +on: + push: + branches: [master, lab05, lab06] + paths: + - 'ansible/**' + - '.github/workflows/ansible-deploy.yml' + pull_request: + branches: [master, lab05, lab06] + paths: + - 'ansible/**' + - '.github/workflows/ansible-deploy.yml' + +jobs: + lint: + name: Ansible Lint + runs-on: ubuntu-latest + defaults: + run: + working-directory: ansible + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.12" + + - name: Install Ansible and collections + run: | + pip install ansible ansible-lint + ansible-galaxy collection install community.docker community.general + + - name: Run ansible-lint + run: ansible-lint playbooks/*.yml + continue-on-error: true + + deploy: + name: Deploy Application + needs: lint + runs-on: ubuntu-latest + if: github.event_name == 'push' && (github.ref == 'refs/heads/master' || github.ref == 'refs/heads/lab06') + defaults: + run: + working-directory: ansible + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.12" + + - name: Install Ansible and collections + run: | + pip install ansible + ansible-galaxy collection install community.docker community.general + + - name: Setup SSH + run: | + mkdir -p ~/.ssh + echo "${{ secrets.SSH_PRIVATE_KEY }}" > ~/.ssh/id_rsa + chmod 600 ~/.ssh/id_rsa + ssh-keyscan -H ${{ secrets.VM_HOST }} >> ~/.ssh/known_hosts 2>/dev/null || true + + - name: Create CI inventory + run: | + cat > inventory/hosts.ci.ini << EOF + [webservers] + deploy-target ansible_host=${{ secrets.VM_HOST }} ansible_user=${{ secrets.VM_USER }} ansible_python_interpreter=/usr/bin/python3 + EOF + + - name: Deploy with Ansible + env: + ANSIBLE_VAULT_PASSWORD: ${{ secrets.ANSIBLE_VAULT_PASSWORD }} + run: | + echo "$ANSIBLE_VAULT_PASSWORD" > /tmp/vault_pass + ansible-playbook playbooks/deploy.yml -i inventory/hosts.ci.ini --vault-password-file /tmp/vault_pass + rm -f /tmp/vault_pass + + - name: Verify deployment + run: | + sleep 10 + curl -sf "http://${{ secrets.VM_HOST }}:5000/health" || exit 1 diff --git a/.github/workflows/go-ci.yml b/.github/workflows/go-ci.yml new file mode 100644 index 0000000000..b74f06240b --- /dev/null +++ b/.github/workflows/go-ci.yml @@ -0,0 +1,82 @@ +name: Go CI + +on: + push: + branches: [master, lab02, lab03] + paths: + - 'app_go/**' + - '.github/workflows/go-ci.yml' + pull_request: + branches: [master, lab02, lab03] + paths: + - 'app_go/**' + - '.github/workflows/go-ci.yml' + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +env: + DOCKER_IMAGE: jambulancia/devops-info-service-go + GO_VERSION: '1.24' + +jobs: + test: + name: Lint & Test + runs-on: ubuntu-latest + defaults: + run: + working-directory: app_go + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Set up Go + uses: actions/setup-go@v5 + with: + go-version: ${{ env.GO_VERSION }} + cache-dependency-path: app_go/go.mod + + - name: Run golangci-lint + uses: golangci/golangci-lint-action@v6 + with: + version: latest + working-directory: app_go + + - name: Run tests + run: go test -v ./... + + docker: + name: Build & Push Docker + runs-on: ubuntu-latest + needs: test + if: github.event_name == 'push' && (github.ref == 'refs/heads/master' || github.ref == 'refs/heads/lab02' || github.ref == 'refs/heads/lab03') + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Log in to Docker Hub + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Generate version + id: meta + run: echo "version=$(date +%Y.%m.%d)" >> $GITHUB_OUTPUT + + - name: Build and push + uses: docker/build-push-action@v6 + with: + context: ./app_go + push: true + tags: | + ${{ env.DOCKER_IMAGE }}:${{ steps.meta.outputs.version }} + ${{ env.DOCKER_IMAGE }}:latest + cache-from: type=gha + cache-to: type=gha,mode=max diff --git a/.github/workflows/python-ci.yml b/.github/workflows/python-ci.yml new file mode 100644 index 0000000000..5c8f2df8d2 --- /dev/null +++ b/.github/workflows/python-ci.yml @@ -0,0 +1,95 @@ +name: Python CI + +on: + push: + branches: [master, lab02, lab03] + paths: + - 'app_python/**' + - '.github/workflows/python-ci.yml' + pull_request: + branches: [master, lab02, lab03] + paths: + - 'app_python/**' + - '.github/workflows/python-ci.yml' + +# Cancel outdated runs +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +env: + DOCKER_IMAGE: jambulancia/devops-info-service + PYTHON_VERSION: '3.13' + +jobs: + test: + name: Lint & Test + runs-on: ubuntu-latest + defaults: + run: + working-directory: app_python + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: ${{ env.PYTHON_VERSION }} + cache: 'pip' + cache-dependency-path: app_python/requirements*.txt + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -r requirements.txt -r requirements-dev.txt + + - name: Run ruff linter + run: ruff check app.py tests/ + + - name: Run tests + run: pytest tests/ -v + + - name: Snyk security scan + uses: snyk/actions/python@master + continue-on-error: true + env: + SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }} + with: + command: test + args: --severity-threshold=high + + docker: + name: Build & Push Docker + runs-on: ubuntu-latest + needs: test + if: github.event_name == 'push' && (github.ref == 'refs/heads/master' || github.ref == 'refs/heads/lab02' || github.ref == 'refs/heads/lab03') + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Log in to Docker Hub + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Generate version + id: meta + run: echo "version=$(date +%Y.%m.%d)" >> $GITHUB_OUTPUT + + - name: Build and push + uses: docker/build-push-action@v6 + with: + context: ./app_python + push: true + tags: | + ${{ env.DOCKER_IMAGE }}:${{ steps.meta.outputs.version }} + ${{ env.DOCKER_IMAGE }}:latest + cache-from: type=gha + cache-to: type=gha,mode=max diff --git a/.github/workflows/terraform-ci.yml b/.github/workflows/terraform-ci.yml new file mode 100644 index 0000000000..965e2e3b0e --- /dev/null +++ b/.github/workflows/terraform-ci.yml @@ -0,0 +1,57 @@ +name: Terraform CI + +on: + push: + branches: [master, lab04] + paths: + - 'terraform/**' + - '.github/workflows/terraform-ci.yml' + pull_request: + branches: [master, lab04] + paths: + - 'terraform/**' + - '.github/workflows/terraform-ci.yml' + +jobs: + validate: + name: Validate Terraform + runs-on: ubuntu-latest + defaults: + run: + working-directory: terraform + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Create dummy SSH public key for validation + run: | + mkdir -p ~/.ssh + echo "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDummyKeyForCIValidationOnly" > ~/.ssh/id_rsa.pub + + - name: Setup Terraform + uses: hashicorp/setup-terraform@v3 + with: + terraform_version: "1.9.0" + terraform_wrapper: false + + - name: Terraform Format Check + run: terraform fmt -check -recursive -diff + + - name: Terraform Init + run: terraform init -backend=false + + - name: Terraform Validate + run: terraform validate + + - name: Setup TFLint + uses: terraform-linters/setup-tflint@v4 + with: + tflint_version: latest + + - name: TFLint Init + run: tflint --init + + - name: TFLint + run: tflint --format compact + continue-on-error: true diff --git a/.gitignore b/.gitignore index 30d74d2584..faf7be4834 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,26 @@ -test \ No newline at end of file +test + +# Terraform +*.tfstate +*.tfstate.* +.terraform/ +.terraform.lock.hcl +terraform.tfvars +*.tfvars + +# Pulumi +Pulumi.*.yaml +pulumi/venv/ +pulumi/.venv/ + +# Credentials & secrets +.env +*.pem +*.key +credentials + +# Ansible +*.retry +.vault_pass +ansible/inventory/*.pyc +__pycache__/ diff --git a/ansible/README.md b/ansible/README.md new file mode 100644 index 0000000000..73b175033e --- /dev/null +++ b/ansible/README.md @@ -0,0 +1,34 @@ +# Ansible — Configuration Management + +[![Ansible Deployment](https://github.com/abdughafforzoda/DevOps-Core-Course/actions/workflows/ansible-deploy.yml/badge.svg)](https://github.com/abdughafforzoda/DevOps-Core-Course/actions/workflows/ansible-deploy.yml) + +Role-based Ansible automation for Lab 5–6: system provisioning (common, docker) and application deployment (web_app) via Docker Compose. + +## Quick start + +```bash +# Edit inventory with your VM +vim inventory/hosts.ini + +# Create vault (one-time) +ansible-vault create group_vars/all.yml # use structure from group_vars/all.yml.example + +# Provision +ansible-playbook playbooks/provision.yml + +# Deploy app +ansible-playbook playbooks/deploy.yml --ask-vault-pass +``` + +## Tags (Lab 6) + +```bash +ansible-playbook playbooks/provision.yml --tags "docker" +ansible-playbook playbooks/provision.yml --skip-tags "common" +ansible-playbook playbooks/deploy.yml -e "web_app_wipe=true" --tags web_app_wipe # wipe only +ansible-playbook playbooks/deploy.yml -e "web_app_wipe=true" # clean reinstall +``` + +## CI/CD + +Secrets required: `ANSIBLE_VAULT_PASSWORD`, `SSH_PRIVATE_KEY`, `VM_HOST`, `VM_USER`. See `.github/workflows/ansible-deploy.yml`. diff --git a/ansible/ansible.cfg b/ansible/ansible.cfg new file mode 100644 index 0000000000..8448fb5796 --- /dev/null +++ b/ansible/ansible.cfg @@ -0,0 +1,12 @@ + [defaults] + inventory = inventory/hosts.ini + roles_path = roles + host_key_checking = False + remote_user = ubuntu + retry_files_enabled = False + + [privilege_escalation] + become = True + become_method = sudo + become_user = root + diff --git a/ansible/docs/LAB05.md b/ansible/docs/LAB05.md new file mode 100644 index 0000000000..ee73de9772 --- /dev/null +++ b/ansible/docs/LAB05.md @@ -0,0 +1,69 @@ + # LAB05 — Ansible Fundamentals + + ## 1. Architecture Overview + + - **Ansible version:** _(fill after running `ansible --version`)_ + - **Target VM:** Ubuntu 22.04/24.04 LTS from Lab 4 + - **Structure:** + - `ansible/ansible.cfg` + - `ansible/inventory/hosts.ini` + - `ansible/roles/common`, `docker`, `web_app` (renamed from app_deploy in Lab 6) + - `ansible/playbooks/provision.yml`, `deploy.yml`, `site.yml` + - `ansible/group_vars/all.yml` (vaulted; example in `all.yml.example`) + + Roles are used instead of monolithic playbooks for reusability, clarity, and easier testing. + + ## 2. Roles Documentation + + ### common + - **Purpose:** Base system provisioning (apt cache, common packages, timezone). + - **Variables:** `common_packages`, `common_timezone`. + - **Handlers:** none. + - **Dependencies:** none. + + ### docker + - **Purpose:** Install and configure Docker Engine and dependencies. + - **Variables:** `docker_packages`, `docker_user`. + - **Handlers:** `restart docker`. + - **Dependencies:** expects `docker_user` to exist (e.g. created outside or by another role). + + ### web_app (formerly app_deploy) + - **Purpose:** Deploy app via Docker Compose; log in to Docker Hub, template compose file, run containers, verify health. + - **Variables:** `app_name`, `app_port`, `app_container_name`, `app_restart_policy`, `app_environment`, plus vaulted `dockerhub_username`, `dockerhub_password`, `docker_image`, `docker_image_tag`. + - **Handlers:** `restart app container`. + - **Dependencies:** Docker installed and running (via `docker` role). + + ## 3. Idempotency Demonstration + + Paste and briefly annotate your outputs: + + - **First run of `playbooks/provision.yml`:** _(expect many `changed`)_ + - **Second run of `playbooks/provision.yml`:** _(expect all `ok`, no `changed`)_ + + Explain which tasks changed on the first run and why nothing changed on the second run (desired state already reached). + + ## 4. Ansible Vault Usage + + - Sensitive values (Docker Hub credentials, image name) are stored in `group_vars/all.yml`, which **you create with `ansible-vault`** using the structure from `group_vars/all.yml.example`. + - Use either `--ask-vault-pass` or a `.vault_pass` file (added to `.gitignore`) for automation. + - Vault ensures credentials are encrypted at rest in Git. + + Show: + - Example of encrypted `group_vars/all.yml` (header only; content unreadable). + - How you manage the vault password. + + ## 5. Deployment Verification + + After running: + + ```bash + ansible-playbook playbooks/deploy.yml --ask-vault-pass + ``` + + Capture: + - `docker ps` on the VM showing the container running. + - `curl http://:5000/health` and `/` outputs. + - Any handler executions (e.g., app restart). + + + diff --git a/ansible/docs/LAB06.md b/ansible/docs/LAB06.md new file mode 100644 index 0000000000..a20a5ee943 --- /dev/null +++ b/ansible/docs/LAB06.md @@ -0,0 +1,114 @@ +# Lab 6: Advanced Ansible & CI/CD + +## 1. Overview + +- **Technologies:** Ansible 2.16+, Docker Compose v2, GitHub Actions, Jinja2 +- **Changes from Lab 5:** + - Roles refactored with blocks, rescue, always, and tags + - `app_deploy` renamed to `web_app` + - Deployment switched from `docker run` to Docker Compose + - Wipe logic with variable + tag safety + - Ansible CI/CD workflow (lint + deploy) +- **Structure:** Same `ansible/` layout; `web_app` uses `templates/docker-compose.yml.j2` and `tasks/wipe.yml` + +--- + +## 2. Blocks & Tags + +### common role +- **Block:** Package installation (apt cache, common packages) with tag `packages` +- **Rescue:** Retry `apt update` and package install on failure +- **Always:** Log completion to `/tmp/ansible_common_complete` +- **Tags:** `packages`, `common` + +### docker role +- **Block 1 (docker_install):** Prerequisites, GPG key, repo, Docker packages +- **Rescue:** Wait 10s, retry apt update and Docker install +- **Always:** Ensure Docker service is enabled and started +- **Block 2 (docker_config):** Add user to docker group, install python3-docker +- **Tags:** `docker_install`, `docker_config`, `docker` + +### web_app role +- **Tags:** `app_deploy`, `compose`, `web_app_wipe` (wipe tasks only) + +### Example commands +```bash +ansible-playbook playbooks/provision.yml --tags "docker" +ansible-playbook playbooks/provision.yml --skip-tags "common" +ansible-playbook playbooks/provision.yml --tags "packages" +ansible-playbook playbooks/provision.yml --tags "docker_install" +ansible-playbook playbooks/provision.yml --list-tags +``` + +--- + +## 3. Docker Compose Migration + +- **Template:** `roles/web_app/templates/docker-compose.yml.j2` + - Uses Jinja2 for `app_name`, `docker_image`, `docker_tag`, `app_port`, `app_internal_port`, `app_environment` + - Restart policy: `unless-stopped` +- **Role dependency:** `roles/web_app/meta/main.yml` declares dependency on `docker` +- **Tasks:** Create app dir, template compose file, `docker_login`, `docker_compose_v2` (state: present, pull: always) +- **App dir:** `/opt/{{ app_name }}` (e.g. `/opt/devops-info-service`) + +--- + +## 4. Wipe Logic + +- **Variable:** `web_app_wipe` (default: `false`) +- **Tag:** `web_app_wipe` +- **Location:** `roles/web_app/tasks/wipe.yml`, included at top of `main.yml` +- **Behavior:** Wipe runs only when `web_app_wipe | bool` is true and tasks with tag `web_app_wipe` are executed +- **Tasks:** `docker compose down`, remove compose file, remove app directory, debug log + +### Test scenarios +| Scenario | Command | Result | +|----------|---------|--------| +| Normal deploy | `ansible-playbook deploy.yml` | Deploy only; wipe skipped | +| Wipe only | `ansible-playbook deploy.yml -e "web_app_wipe=true" --tags web_app_wipe` | Wipe only; deploy skipped | +| Clean reinstall | `ansible-playbook deploy.yml -e "web_app_wipe=true"` | Wipe then deploy | +| Tag only, var false | `ansible-playbook deploy.yml --tags web_app_wipe` | Wipe skipped (when blocks it) | + +--- + +## 5. CI/CD Integration + +- **Workflow:** `.github/workflows/ansible-deploy.yml` +- **Triggers:** Push/PR to `ansible/**` on master, lab05, lab06 +- **Jobs:** + 1. **lint:** ansible-lint on playbooks (continue-on-error: true) + 2. **deploy:** Runs only on push to master/lab06; requires secrets +- **Secrets:** `ANSIBLE_VAULT_PASSWORD`, `SSH_PRIVATE_KEY`, `VM_HOST`, `VM_USER` +- **Deploy steps:** Install Ansible, setup SSH, create CI inventory, run `deploy.yml` with vault, verify `/health` +- **Verification:** `curl http://VM_HOST:5000/health` after deploy + +### Badge +```markdown +[![Ansible Deployment](https://github.com/abdughafforzoda/DevOps-Core-Course/actions/workflows/ansible-deploy.yml/badge.svg)](https://github.com/abdughafforzoda/DevOps-Core-Course/actions/workflows/ansible-deploy.yml) +``` + +--- + +## 6. Testing + +- **Idempotency:** Run `deploy.yml` twice; second run should show mostly `ok`, no changes. +- **Selective tags:** Use `--tags` and `--skip-tags` as in section 2. +- **Wipe tests:** Run all four scenarios in section 4 and verify. +- **CI:** Push changes to `ansible/`, confirm workflow runs and lint passes; deploy passes when secrets are set. + +--- + +## 7. Challenges + +- _(Add any issues and how you resolved them)_ +- **Note:** `community.docker.docker_compose_v2` has no `state: restarted`; handler uses `docker compose restart` via `command` module. + +--- + +## 8. Research Answers + +1. **Variable + tag:** Variable ensures wipe is explicit; tag limits wipe to runs where wipe is intended. Prevents accidental wipe. +2. **`never` vs this approach:** `never` runs only when explicitly requested; our approach also requires the variable. +3. **Wipe before deploy:** Enables wipe → deploy in one run (clean reinstall). +4. **Clean reinstall vs rolling update:** Clean reinstall for major changes or corruption; rolling update for low-risk updates. +5. **Extending wipe:** Add tasks to remove images (`docker image prune`) and volumes (`docker volume rm`) after `compose down`. diff --git a/ansible/group_vars/all.yml.example b/ansible/group_vars/all.yml.example new file mode 100644 index 0000000000..04efebb10a --- /dev/null +++ b/ansible/group_vars/all.yml.example @@ -0,0 +1,15 @@ +--- +# Docker Hub credentials (use ansible-vault for real file) +dockerhub_username: your-username +dockerhub_password: your-access-token + +# Application configuration +app_name: devops-info-service +docker_image: "{{ dockerhub_username }}/{{ app_name }}" +docker_image_tag: latest +docker_tag: "{{ docker_image_tag }}" +app_port: 5000 +app_internal_port: 5000 +app_container_name: "{{ app_name }}" +compose_project_dir: "/opt/{{ app_name }}" + diff --git a/ansible/inventory/hosts.ini b/ansible/inventory/hosts.ini new file mode 100644 index 0000000000..75fa0f5aef --- /dev/null +++ b/ansible/inventory/hosts.ini @@ -0,0 +1,3 @@ + [webservers] + vm-name ansible_host=
ansible_user=ubuntu ansible_python_interpreter=/usr/bin/python3 + diff --git a/ansible/playbooks/deploy.yml b/ansible/playbooks/deploy.yml new file mode 100644 index 0000000000..d91d1dab28 --- /dev/null +++ b/ansible/playbooks/deploy.yml @@ -0,0 +1,9 @@ +--- +- name: Deploy application + hosts: webservers + become: true + + roles: + - role: web_app + tags: [web_app, app_deploy, compose] + diff --git a/ansible/playbooks/provision.yml b/ansible/playbooks/provision.yml new file mode 100644 index 0000000000..a2e6f74cd5 --- /dev/null +++ b/ansible/playbooks/provision.yml @@ -0,0 +1,11 @@ +--- +- name: Provision web servers + hosts: webservers + become: true + + roles: + - role: common + tags: [common, packages] + - role: docker + tags: [docker, docker_install, docker_config] + diff --git a/ansible/playbooks/site.yml b/ansible/playbooks/site.yml new file mode 100644 index 0000000000..5a3c203ebd --- /dev/null +++ b/ansible/playbooks/site.yml @@ -0,0 +1,10 @@ + --- + - name: Full provisioning and deployment + hosts: webservers + become: true + + roles: + - common + - docker + - web_app + diff --git a/ansible/roles/common/defaults/main.yml b/ansible/roles/common/defaults/main.yml new file mode 100644 index 0000000000..2027a15cc5 --- /dev/null +++ b/ansible/roles/common/defaults/main.yml @@ -0,0 +1,15 @@ + --- + # Default packages for common system setup + common_packages: + - python3-pip + - curl + - git + - vim + - htop + - ca-certificates + - apt-transport-https + + common_timezone: Etc/UTC + # User to ensure exists for deployments (optional) + common_deploy_user: null + diff --git a/ansible/roles/common/tasks/main.yml b/ansible/roles/common/tasks/main.yml new file mode 100644 index 0000000000..0d1e801fa0 --- /dev/null +++ b/ansible/roles/common/tasks/main.yml @@ -0,0 +1,60 @@ +--- +- name: Install common packages + block: + - name: Update apt cache + ansible.builtin.apt: + update_cache: true + cache_valid_time: 3600 + + - name: Install common packages + ansible.builtin.apt: + name: "{{ common_packages }}" + state: present + install_recommends: false + + rescue: + - name: Retry apt update on failure + ansible.builtin.apt: + update_cache: true + cache_valid_time: 0 + + - name: Retry package installation + ansible.builtin.apt: + name: "{{ common_packages }}" + state: present + install_recommends: false + + always: + - name: Log common role completion + ansible.builtin.copy: + content: "common role completed at {{ ansible_date_time.iso8601 }}\n" + dest: /tmp/ansible_common_complete + mode: "0644" + + tags: + - packages + - common + +- name: Ensure deploy user exists + block: + - name: Create deploy user + ansible.builtin.user: + name: "{{ common_deploy_user }}" + state: present + shell: /bin/bash + create_home: true + + when: common_deploy_user is defined and common_deploy_user | length > 0 + tags: + - users + - common + +- name: Set system timezone + block: + - name: Configure timezone + community.general.timezone: + name: "{{ common_timezone }}" + + when: common_timezone is defined + tags: + - common diff --git a/ansible/roles/docker/defaults/main.yml b/ansible/roles/docker/defaults/main.yml new file mode 100644 index 0000000000..c7f41e60fc --- /dev/null +++ b/ansible/roles/docker/defaults/main.yml @@ -0,0 +1,8 @@ + --- + docker_packages: + - docker-ce + - docker-ce-cli + - containerd.io + + docker_user: ubuntu + diff --git a/ansible/roles/docker/handlers/main.yml b/ansible/roles/docker/handlers/main.yml new file mode 100644 index 0000000000..0fbc85062b --- /dev/null +++ b/ansible/roles/docker/handlers/main.yml @@ -0,0 +1,6 @@ + --- + - name: restart docker + ansible.builtin.service: + name: docker + state: restarted + diff --git a/ansible/roles/docker/tasks/main.yml b/ansible/roles/docker/tasks/main.yml new file mode 100644 index 0000000000..bf5f2148a6 --- /dev/null +++ b/ansible/roles/docker/tasks/main.yml @@ -0,0 +1,86 @@ +--- +- name: Install Docker + block: + - name: Install prerequisite packages + ansible.builtin.apt: + name: + - ca-certificates + - curl + - gnupg + state: present + update_cache: true + + - name: Add Docker GPG key + ansible.builtin.apt_key: + url: https://download.docker.com/linux/ubuntu/gpg + state: present + + - name: Add Docker APT repository + ansible.builtin.apt_repository: + repo: "deb [arch=amd64] https://download.docker.com/linux/ubuntu {{ ansible_distribution_release }} stable" + state: present + notify: restart docker + + - name: Install Docker packages + ansible.builtin.apt: + name: "{{ docker_packages }}" + state: present + update_cache: true + notify: restart docker + + rescue: + - name: Wait before retry + ansible.builtin.pause: + seconds: 10 + prompt: "Retrying after GPG/repo failure..." + + - name: Retry apt update + ansible.builtin.apt: + update_cache: true + cache_valid_time: 0 + + - name: Retry Docker GPG key + ansible.builtin.apt_key: + url: https://download.docker.com/linux/ubuntu/gpg + state: present + + - name: Retry Docker repository + ansible.builtin.apt_repository: + repo: "deb [arch=amd64] https://download.docker.com/linux/ubuntu {{ ansible_distribution_release }} stable" + state: present + notify: restart docker + + - name: Retry Docker package install + ansible.builtin.apt: + name: "{{ docker_packages }}" + state: present + update_cache: true + notify: restart docker + + always: + - name: Ensure Docker service is enabled and started + ansible.builtin.service: + name: docker + state: started + enabled: true + + tags: + - docker_install + - docker + +- name: Configure Docker + block: + - name: Add user to docker group + ansible.builtin.user: + name: "{{ docker_user }}" + groups: docker + append: true + + - name: Install python3-docker for Ansible Docker modules + ansible.builtin.apt: + name: python3-docker + state: present + + tags: + - docker_config + - docker diff --git a/ansible/roles/web_app/defaults/main.yml b/ansible/roles/web_app/defaults/main.yml new file mode 100644 index 0000000000..b390d49f0f --- /dev/null +++ b/ansible/roles/web_app/defaults/main.yml @@ -0,0 +1,18 @@ +--- +app_name: devops-info-service +docker_image: "{{ dockerhub_username }}/{{ app_name }}" +docker_tag: "{{ docker_image_tag | default('latest') }}" +app_port: 5000 +app_internal_port: 5000 +app_container_name: "{{ app_name }}" +app_restart_policy: unless-stopped +app_environment: {} + +# Compose config +compose_project_dir: "/opt/{{ app_name }}" +docker_compose_version: "3.8" + +# Wipe logic: set to true to remove application completely +# Wipe only: ansible-playbook deploy.yml -e "web_app_wipe=true" --tags web_app_wipe +# Clean install: ansible-playbook deploy.yml -e "web_app_wipe=true" +web_app_wipe: false diff --git a/ansible/roles/web_app/handlers/main.yml b/ansible/roles/web_app/handlers/main.yml new file mode 100644 index 0000000000..c325f998c9 --- /dev/null +++ b/ansible/roles/web_app/handlers/main.yml @@ -0,0 +1,5 @@ +--- +- name: restart app container + ansible.builtin.command: + cmd: docker compose restart + chdir: "{{ compose_project_dir }}" diff --git a/ansible/roles/web_app/meta/main.yml b/ansible/roles/web_app/meta/main.yml new file mode 100644 index 0000000000..4990033a1e --- /dev/null +++ b/ansible/roles/web_app/meta/main.yml @@ -0,0 +1,4 @@ +--- +# Ensure Docker is installed before deploying the web app +dependencies: + - role: docker diff --git a/ansible/roles/web_app/tasks/main.yml b/ansible/roles/web_app/tasks/main.yml new file mode 100644 index 0000000000..de0d9400fe --- /dev/null +++ b/ansible/roles/web_app/tasks/main.yml @@ -0,0 +1,51 @@ +--- +# Wipe logic runs first when explicitly requested (-e web_app_wipe=true) +- name: Include wipe tasks + ansible.builtin.include_tasks: wipe.yml + tags: + - web_app_wipe + +# Deployment block: skip when wipe-only (tag web_app_wipe alone) +- name: Deploy application with Docker Compose + block: + - name: Create app directory + ansible.builtin.file: + path: "{{ compose_project_dir }}" + state: directory + mode: "0755" + + - name: Template docker-compose file + ansible.builtin.template: + src: docker-compose.yml.j2 + dest: "{{ compose_project_dir }}/docker-compose.yml" + mode: "0644" + notify: restart app container + + - name: Log in to Docker Hub + community.docker.docker_login: + username: "{{ dockerhub_username }}" + password: "{{ dockerhub_password }}" + no_log: true + + - name: Deploy with Docker Compose + community.docker.docker_compose_v2: + project_src: "{{ compose_project_dir }}" + state: present + pull: always + + - name: Wait for application port on target + ansible.builtin.wait_for: + host: 127.0.0.1 + port: "{{ app_port }}" + delay: 5 + timeout: 60 + + - name: Check health endpoint + ansible.builtin.uri: + url: "http://127.0.0.1:{{ app_port }}/health" + method: GET + status_code: 200 + + tags: + - app_deploy + - compose diff --git a/ansible/roles/web_app/tasks/wipe.yml b/ansible/roles/web_app/tasks/wipe.yml new file mode 100644 index 0000000000..c3341d2372 --- /dev/null +++ b/ansible/roles/web_app/tasks/wipe.yml @@ -0,0 +1,29 @@ +--- +# Wipe logic: controlled by web_app_wipe variable + web_app_wipe tag +# Only runs when both -e "web_app_wipe=true" AND --tags web_app_wipe (or full deploy with wipe) +- name: Wipe web application + block: + - name: Stop and remove containers + community.docker.docker_compose_v2: + project_src: "{{ compose_project_dir }}" + state: absent + ignore_errors: true + + - name: Remove docker-compose file + ansible.builtin.file: + path: "{{ compose_project_dir }}/docker-compose.yml" + state: absent + ignore_errors: true + + - name: Remove application directory + ansible.builtin.file: + path: "{{ compose_project_dir }}" + state: absent + + - name: Log wipe completion + ansible.builtin.debug: + msg: "Application {{ app_name }} wiped successfully" + + when: web_app_wipe | default(false) | bool + tags: + - web_app_wipe diff --git a/ansible/roles/web_app/templates/docker-compose.yml.j2 b/ansible/roles/web_app/templates/docker-compose.yml.j2 new file mode 100644 index 0000000000..852da84be4 --- /dev/null +++ b/ansible/roles/web_app/templates/docker-compose.yml.j2 @@ -0,0 +1,15 @@ +version: '{{ docker_compose_version }}' + +services: + {{ app_name }}: + image: {{ docker_image }}:{{ docker_tag }} + container_name: {{ app_container_name }} + ports: + - "{{ app_port }}:{{ app_internal_port }}" + environment: + PORT: "{{ app_internal_port }}" + HOST: "0.0.0.0" + {% for key, value in app_environment.items() %} + {{ key }}: "{{ value }}" + {% endfor %} + restart: {{ app_restart_policy }} diff --git a/app_go/.dockerignore b/app_go/.dockerignore new file mode 100644 index 0000000000..6ff436b9f9 --- /dev/null +++ b/app_go/.dockerignore @@ -0,0 +1,21 @@ +# Version control +.git/ +.gitignore + +# IDE / editor +.vscode/ +.idea/ +*.swp +*.swo + +# OS +.DS_Store +Thumbs.db + +docs/ +*.md +README* + +*.exe +*.test +*.out diff --git a/app_go/.gitignore b/app_go/.gitignore new file mode 100644 index 0000000000..94b0741a64 --- /dev/null +++ b/app_go/.gitignore @@ -0,0 +1,7 @@ +# Binaries +devops-info-service +devops-info-service-small +*.exe + +# OS +.DS_Store diff --git a/app_go/Dockerfile b/app_go/Dockerfile new file mode 100644 index 0000000000..d5719890e8 --- /dev/null +++ b/app_go/Dockerfile @@ -0,0 +1,32 @@ +# DevOps Info Service — Go (Lab 2 Bonus) +# Multi-stage build: separate build environment from runtime + +FROM golang:1.24-alpine AS builder + +WORKDIR /build + +COPY go.mod . +RUN go mod download + +COPY main.go . + +RUN CGO_ENABLED=0 GOOS=linux go build -ldflags="-s -w" -o devops-info-service . + +FROM alpine:3.21 + +RUN addgroup -g 1000 appgroup && \ + adduser -D -u 1000 -G appgroup appuser + +WORKDIR /app + +COPY --from=builder /build/devops-info-service . + +RUN chown appuser:appgroup devops-info-service + +USER appuser + +EXPOSE 5000 + +ENV PORT=5000 HOST=0.0.0.0 + +CMD ["./devops-info-service"] diff --git a/app_go/README.md b/app_go/README.md new file mode 100644 index 0000000000..646594edca --- /dev/null +++ b/app_go/README.md @@ -0,0 +1,82 @@ +# DevOps Info Service (Go) + +[![Go CI](https://github.com/abdughafforzoda/DevOps-Core-Course/actions/workflows/go-ci.yml/badge.svg)](https://github.com/abdughafforzoda/DevOps-Core-Course/actions/workflows/go-ci.yml) + +Go implementation of the DevOps Info Service — same endpoints and JSON structure as the Python version. Used for Lab 1 bonus and as a basis for multi-stage Docker builds in Lab 2. + +## Prerequisites + +- **Go 1.21+** (1.24 used during development) + +## Build + +```bash +cd app_go + +# Standard build +go build -o devops-info-service . + +# Smaller binary (strip debug info) — recommended for Docker +go build -ldflags="-s -w" -o devops-info-service . +``` + +## Run + +```bash +./devops-info-service +``` + +Defaults: `HOST=0.0.0.0`, `PORT=5000`. + +Custom config: + +```bash +PORT=8080 ./devops-info-service +HOST=127.0.0.1 PORT=3000 ./devops-info-service +``` + +## API Endpoints + +- **`GET /`** — Service, system, runtime, and request info + endpoints list. +- **`GET /health`** — Health check (`status`, `timestamp`, `uptime_seconds`). + +## Configuration + +| Variable | Default | Description | +|----------|-----------|--------------------| +| `HOST` | `0.0.0.0` | Bind address | +| `PORT` | `5000` | Listen port | + +## Binary size vs Python + +| Build | Size | +|-------|--------| +| `go build` (default) | ~8.1 MB | +| `go build -ldflags="-s -w"` | ~5.5 MB | + +Python runs via interpreter + virtualenv; there is no single executable. The Go binary is self-contained and suitable for minimal Docker images (e.g. `scratch` or `alpine`). + +## Test + +```bash +cd app_go +go test -v ./... +``` + +## Docker + +Multi-stage build (Lab 2 bonus): + +```bash +docker build -t devops-info-service-go . +docker run -p 5000:5000 devops-info-service-go +``` + +See `docs/LAB02.md` for the multi-stage strategy and size analysis. + +## Test + +```bash +curl -s http://localhost:5000/ | jq +curl -s http://localhost:5000/health | jq +``` diff --git a/app_go/docs/GO.md b/app_go/docs/GO.md new file mode 100644 index 0000000000..11e7287a1e --- /dev/null +++ b/app_go/docs/GO.md @@ -0,0 +1,21 @@ +# Go — Language Justification + +## Why Go for this service + +- **Small, static binaries** — Single executable, no runtime or interpreter. Ideal for Docker multi-stage builds (Lab 2) and minimal images (`scratch` / `alpine`). +- **Fast compilation** — `go build` completes in seconds. Good for CI/CD. +- **Standard library** — `net/http`, `encoding/json`, `os`, `runtime` cover everything we need. No external dependencies. +- **Simple concurrency** — Goroutines and channels are available if we add more workloads later; for this lab, a single handler is enough. +- **Tooling** — `go build`, `go test`, `go mod` are built-in and straightforward. + +## Compared to alternatives + +| Criterion | Go | Rust | Java/Spring Boot | C# / ASP.NET Core | +|------------------|----------|-------------|------------------|-------------------| +| Binary size | ~5–8 MB | Similar | Large (JVM) | Large (.NET) | +| Build speed | Very fast| Slower | Moderate | Moderate | +| Learning curve | Low | Steep | Moderate | Moderate | +| Stdlib HTTP | Yes | Via crates | Via framework | Via framework | +| Lab 2 Docker fit | Excellent| Good | Heavier | Heavier | + +Go is a good fit for a small HTTP service that will be containerized and used in CI/CD and Kubernetes later in the course. diff --git a/app_go/docs/LAB01.md b/app_go/docs/LAB01.md new file mode 100644 index 0000000000..1698e252a8 --- /dev/null +++ b/app_go/docs/LAB01.md @@ -0,0 +1,52 @@ +# Lab 1 Bonus — Go Implementation + +## Overview + +Same DevOps Info Service as the Python app: `GET /` (service + system + runtime + request + endpoints) and `GET /health` (health check). Implemented in Go using only the standard library. + +## Implementation details + +- **`main.go`** — Single binary. Handlers: `mainHandler` for `/`, `healthHandler` for `/health`. `/health` is registered first so it is matched before `/`. +- **System info** — `os.Hostname()`, `runtime.GOOS` / `GOARCH` / `NumCPU()` / `Version()`. `platform_version` comes from `/etc/os-release` (`PRETTY_NAME`) on Linux; otherwise `runtime.GOOS`. +- **Uptime** — `startTime` stored at startup; duration computed on each request. Same human-readable format as Python (`"X hours, Y minutes"`). +- **Request info** — Client IP from `RemoteAddr` (or `X-Forwarded-For`), `User-Agent`, method, path. +- **Config** — `HOST` and `PORT` via env; defaults `0.0.0.0` and `5000`. + +## JSON structure + +Matches the Python shape. Differences: + +- **`system.go_version`** — Go version (e.g. `1.24.2`) instead of `python_version`. +- **`service.framework`** — `"net/http"` (stdlib) instead of `"Flask"`. + +## Build and run + +```bash +cd app_go +go build -ldflags="-s -w" -o devops-info-service . +./devops-info-service +``` + +```bash +curl -s http://localhost:5000/ | jq +curl -s http://localhost:5000/health | jq +``` + +## Binary size vs Python + +| Build | Size (approx.) | +|-------|----------------| +| `go build` | ~8.1 MB | +| `go build -ldflags="-s -w"` | ~5.5 MB | + +Python runs via interpreter + venv; there is no single executable. The Go binary is self-contained and suitable for minimal container images. + +## Screenshots + +Place in `app_go/docs/screenshots/`: + +- Main endpoint (`GET /`) — full JSON. +- Health check (`GET /health`) — JSON response. +- Formatted output (e.g. `curl … | jq`) or browser. + +Build and run the binary, then capture these to complete the bonus submission. diff --git a/app_go/docs/LAB02.md b/app_go/docs/LAB02.md new file mode 100644 index 0000000000..8ad23ea145 --- /dev/null +++ b/app_go/docs/LAB02.md @@ -0,0 +1,109 @@ +## LAB02 — Docker Containerization (Go, Bonus) + +### Multi-Stage Build Strategy + +The Go app is containerized using a **multi-stage Dockerfile** with two stages: + +1. **Builder stage** (`golang:1.24-alpine`): Compiles the application. +2. **Runtime stage** (`alpine:3.21`): Runs only the compiled binary. + +**Why multi-stage?** The builder image includes the Go compiler, SDK, and build tools (~300 MB). The runtime image needs none of that—just the static binary and a minimal OS. Copying only the binary into Alpine yields an image under 20 MB. + +--- + +### Stage Breakdown + +#### Stage 1: Builder + +```dockerfile +FROM golang:1.24-alpine AS builder +WORKDIR /build +COPY go.mod . +RUN go mod download +COPY main.go . +RUN CGO_ENABLED=0 GOOS=linux go build -ldflags="-s -w" -o devops-info-service . +``` + +| Step | Purpose | +|------|---------| +| `go.mod` first | Dependencies are cached when only `main.go` changes | +| `CGO_ENABLED=0` | Produces a static binary with no C dependencies; works on any Linux base | +| `-ldflags="-s -w"` | Strips debug info to reduce binary size | +| `-o devops-info-service` | Single output binary to copy into runtime | + +#### Stage 2: Runtime + +```dockerfile +FROM alpine:3.21 +RUN addgroup -g 1000 appgroup && adduser -D -u 1000 -G appgroup appuser +WORKDIR /app +COPY --from=builder /build/devops-info-service . +RUN chown appuser:appgroup devops-info-service +USER appuser +EXPOSE 5000 +ENV PORT=5000 HOST=0.0.0.0 +CMD ["./devops-info-service"] +``` + +| Step | Purpose | +|------|---------| +| `COPY --from=builder` | Copy only the binary from the builder stage | +| Non-root user | Run as `appuser` for security | +| Alpine 3.21 | Small base (~5 MB); static binary needs no C runtime | + +--- + +### Size Comparison + +*Replace with your actual output from `docker images`.* + +| Image | Size | +|-------|------| +| `golang:1.24-alpine` (builder) | ~300 MB | +| `devops-info-service` (final) | ~15–20 MB | + +**Size reduction:** ~95% smaller than using the builder image as the final image. + +--- + +### Why Multi-Stage Matters for Compiled Languages + +- **Builder image:** Includes compiler, linker, headers, and libraries. Necessary only for building. +- **Runtime:** Only needs the compiled binary and minimal runtime (Alpine). +- **Security:** Fewer packages and tools reduce attack surface. +- **Deploy speed:** Smaller images pull and start faster. + +--- + +### Build & Run + +**Build:** + +```bash +cd app_go +docker build -t devops-info-service-go . +``` + +*Add your actual build output here.* + +**Run:** + +```bash +docker run -d -p 5000:5000 --name devops-go devops-info-service-go +``` + +**Test:** + +```bash +curl http://localhost:5000/ +curl http://localhost:5000/health +``` + +--- + +### Security Benefits + +- **Non-root user:** Limits damage if the app is compromised. +- **Minimal base:** Alpine has fewer packages than full distros. +- **Static binary:** No runtime dependency installation; fewer paths for supply-chain issues. +- **Smaller image:** Less code to audit and fewer CVE-prone components. diff --git a/app_go/docs/screenshots/Screenshot From 2026-01-28 22-23-49.png b/app_go/docs/screenshots/Screenshot From 2026-01-28 22-23-49.png new file mode 100644 index 0000000000..f696d3cf53 Binary files /dev/null and b/app_go/docs/screenshots/Screenshot From 2026-01-28 22-23-49.png differ diff --git a/app_go/docs/screenshots/Screenshot From 2026-01-28 22-23-59.png b/app_go/docs/screenshots/Screenshot From 2026-01-28 22-23-59.png new file mode 100644 index 0000000000..3591e381c0 Binary files /dev/null and b/app_go/docs/screenshots/Screenshot From 2026-01-28 22-23-59.png differ diff --git a/app_go/docs/screenshots/Screenshot From 2026-01-28 22-24-55.png b/app_go/docs/screenshots/Screenshot From 2026-01-28 22-24-55.png new file mode 100644 index 0000000000..545469177d Binary files /dev/null and b/app_go/docs/screenshots/Screenshot From 2026-01-28 22-24-55.png differ diff --git a/app_go/docs/screenshots/Screenshot From 2026-02-04 16-44-48.png b/app_go/docs/screenshots/Screenshot From 2026-02-04 16-44-48.png new file mode 100644 index 0000000000..0e50777303 Binary files /dev/null and b/app_go/docs/screenshots/Screenshot From 2026-02-04 16-44-48.png differ diff --git a/app_go/go.mod b/app_go/go.mod new file mode 100644 index 0000000000..85ee40ae7d --- /dev/null +++ b/app_go/go.mod @@ -0,0 +1,3 @@ +module devops-info-service + +go 1.24.2 diff --git a/app_go/main.go b/app_go/main.go new file mode 100644 index 0000000000..2fa886a0b5 --- /dev/null +++ b/app_go/main.go @@ -0,0 +1,197 @@ +// DevOps Info Service — Go implementation. +// Same endpoints and JSON structure as the Python version. + +package main + +import ( + "bufio" + "encoding/json" + "fmt" + "log" + "net" + "net/http" + "os" + "runtime" + "strings" + "time" +) + +type ServiceInfo struct { + Service Service `json:"service"` + System System `json:"system"` + Runtime Runtime `json:"runtime"` + Request Request `json:"request"` + Endpoints []Endpoint `json:"endpoints"` +} + +type Service struct { + Name string `json:"name"` + Version string `json:"version"` + Description string `json:"description"` + Framework string `json:"framework"` +} + +type System struct { + Hostname string `json:"hostname"` + Platform string `json:"platform"` + PlatformVersion string `json:"platform_version"` + Architecture string `json:"architecture"` + CPUCount int `json:"cpu_count"` + GoVersion string `json:"go_version"` +} + +type Runtime struct { + UptimeSeconds int `json:"uptime_seconds"` + UptimeHuman string `json:"uptime_human"` + CurrentTime string `json:"current_time"` + Timezone string `json:"timezone"` +} + +type Request struct { + ClientIP string `json:"client_ip"` + UserAgent string `json:"user_agent"` + Method string `json:"method"` + Path string `json:"path"` +} + +type Endpoint struct { + Path string `json:"path"` + Method string `json:"method"` + Description string `json:"description"` +} + +type HealthResponse struct { + Status string `json:"status"` + Timestamp string `json:"timestamp"` + UptimeSeconds int `json:"uptime_seconds"` +} + +var startTime = time.Now().UTC() + +func getHostname() string { + h, err := os.Hostname() + if err != nil { + return "unknown" + } + return h +} + +func getPlatformVersion() string { + f, err := os.Open("/etc/os-release") + if err != nil { + return runtime.GOOS + } + defer f.Close() + s := bufio.NewScanner(f) + for s.Scan() { + line := strings.TrimSpace(s.Text()) + if strings.HasPrefix(line, "PRETTY_NAME=") { + v := strings.TrimPrefix(line, "PRETTY_NAME=") + v = strings.Trim(v, "\"") + return v + } + } + return runtime.GOOS +} + +func uptime() (seconds int, human string) { + d := time.Since(startTime) + sec := int(d.Seconds()) + h := sec / 3600 + m := (sec % 3600) / 60 + return sec, fmt.Sprintf("%d hours, %d minutes", h, m) +} + +func nowUTC() string { + return time.Now().UTC().Format("2006-01-02T15:04:05.000Z") +} + +func clientIP(r *http.Request) string { + ra := r.RemoteAddr + if h := r.Header.Get("X-Forwarded-For"); h != "" { + if idx := strings.Index(h, ","); idx > 0 { + ra = strings.TrimSpace(h[:idx]) + } else { + ra = strings.TrimSpace(h) + } + return ra + } + host, _, err := net.SplitHostPort(ra) + if err != nil { + return ra + } + return host +} + +func mainHandler(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/" { + http.NotFound(w, r) + return + } + log.Printf("Handling GET /") + + sec, human := uptime() + info := ServiceInfo{ + Service: Service{ + Name: "devops-info-service", + Version: "1.0.0", + Description: "DevOps course info service", + Framework: "net/http", + }, + System: System{ + Hostname: getHostname(), + Platform: runtime.GOOS, + PlatformVersion: getPlatformVersion(), + Architecture: runtime.GOARCH, + CPUCount: runtime.NumCPU(), + GoVersion: strings.TrimPrefix(runtime.Version(), "go"), + }, + Runtime: Runtime{ + UptimeSeconds: sec, + UptimeHuman: human, + CurrentTime: nowUTC(), + Timezone: "UTC", + }, + Request: Request{ + ClientIP: clientIP(r), + UserAgent: r.Header.Get("User-Agent"), + Method: r.Method, + Path: r.URL.Path, + }, + Endpoints: []Endpoint{ + {Path: "/", Method: "GET", Description: "Service information"}, + {Path: "/health", Method: "GET", Description: "Health check"}, + }, + } + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(info) +} + +func healthHandler(w http.ResponseWriter, r *http.Request) { + log.Printf("Handling GET /health") + sec, _ := uptime() + resp := HealthResponse{ + Status: "healthy", + Timestamp: nowUTC(), + UptimeSeconds: sec, + } + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusOK) + _ = json.NewEncoder(w).Encode(resp) +} + +func main() { + port := os.Getenv("PORT") + if port == "" { + port = "5000" + } + host := os.Getenv("HOST") + if host == "" { + host = "0.0.0.0" + } + addr := net.JoinHostPort(host, port) + log.Printf("Starting DevOps Info Service on %s", addr) + http.HandleFunc("/health", healthHandler) + http.HandleFunc("/", mainHandler) + log.Fatal(http.ListenAndServe(addr, nil)) +} diff --git a/app_go/main_test.go b/app_go/main_test.go new file mode 100644 index 0000000000..43f4170a9a --- /dev/null +++ b/app_go/main_test.go @@ -0,0 +1,67 @@ +package main + +import ( + "net/http" + "net/http/httptest" + "testing" +) + +func setupTestServer() *httptest.Server { + mux := http.NewServeMux() + mux.HandleFunc("/health", healthHandler) + mux.HandleFunc("/", mainHandler) + return httptest.NewServer(mux) +} + +func TestHealthEndpoint(t *testing.T) { + server := setupTestServer() + defer server.Close() + + resp, err := http.Get(server.URL + "/health") + if err != nil { + t.Fatalf("GET /health: %v", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + t.Errorf("expected status 200, got %d", resp.StatusCode) + } + + if ct := resp.Header.Get("Content-Type"); ct != "application/json" { + t.Errorf("expected Content-Type application/json, got %s", ct) + } +} + +func TestMainEndpoint(t *testing.T) { + server := setupTestServer() + defer server.Close() + + resp, err := http.Get(server.URL + "/") + if err != nil { + t.Fatalf("GET /: %v", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + t.Errorf("expected status 200, got %d", resp.StatusCode) + } + + if ct := resp.Header.Get("Content-Type"); ct != "application/json" { + t.Errorf("expected Content-Type application/json, got %s", ct) + } +} + +func Test404Endpoint(t *testing.T) { + server := setupTestServer() + defer server.Close() + + resp, err := http.Get(server.URL + "/nonexistent") + if err != nil { + t.Fatalf("GET /nonexistent: %v", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusNotFound { + t.Errorf("expected status 404, got %d", resp.StatusCode) + } +} diff --git a/app_python/.dockerignore b/app_python/.dockerignore new file mode 100644 index 0000000000..6c5dd5c36e --- /dev/null +++ b/app_python/.dockerignore @@ -0,0 +1,34 @@ +# Python +__pycache__/ +*.py[cod] +*.pyo +*.pyd +*.log +venv/ +.venv/ +.env +.env.* + +# Version control +.git/ +.gitignore + +# IDE / editor +.vscode/ +.idea/ +*.swp +*.swo + +# OS +.DS_Store +Thumbs.db + +docs/ +*.md +README* + +tests/ +pytest.ini +.pytest_cache/ +.coverage +htmlcov/ diff --git a/app_python/.gitignore b/app_python/.gitignore new file mode 100644 index 0000000000..3e627abc88 --- /dev/null +++ b/app_python/.gitignore @@ -0,0 +1,14 @@ +# Python +__pycache__/ +*.py[cod] +*.log +venv/ +.venv/ + +# IDE +.vscode/ +.idea/ + +# OS +.DS_Store + diff --git a/app_python/Dockerfile b/app_python/Dockerfile new file mode 100644 index 0000000000..b0cf63192d --- /dev/null +++ b/app_python/Dockerfile @@ -0,0 +1,23 @@ +FROM python:3.13-slim + +ENV PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 + +WORKDIR /app + +COPY requirements.txt . +RUN pip install --no-cache-dir -r requirements.txt + +RUN groupadd -r appgroup && useradd -r -g appgroup appuser + +COPY app.py . + +RUN chown -R appuser:appgroup /app + +USER appuser + +EXPOSE 5000 + +ENV PORT=5000 + +CMD ["python", "app.py"] diff --git a/app_python/README.md b/app_python/README.md new file mode 100644 index 0000000000..32494beb5d --- /dev/null +++ b/app_python/README.md @@ -0,0 +1,102 @@ +## DevOps Info Service (Python) + +[![Python CI](https://github.com/abdughafforzoda/DevOps-Core-Course/actions/workflows/python-ci.yml/badge.svg)](https://github.com/abdughafforzoda/DevOps-Core-Course/actions/workflows/python-ci.yml) + +### Overview + +This is a simple **DevOps Info Service** implemented in Python using **Flask**. +It exposes HTTP endpoints that return detailed information about the service, the underlying system, and its runtime environment. +The service will be used as a foundation for future labs (containerization, CI/CD, monitoring, and more). + +### Prerequisites + +- **Python**: 3.11 or newer +- **Pip**: Python package manager +- Recommended: virtual environment (`venv`) + +### Installation + +```bash +cd app_python + +python -m venv venv +source venv/bin/activate + +pip install -r requirements.txt +``` + +### Running the Application + +Default configuration (host `0.0.0.0`, port `5000`): + +```bash +python app.py +``` + +Custom configuration using environment variables: + +```bash +PORT=8080 python app.py + +HOST=127.0.0.1 PORT=3000 DEBUG=true python app.py +``` + +### API Endpoints + +- `GET /` + - Returns service metadata, system information, runtime information, request details, and a list of available endpoints. +- `GET /health` + - Simple health check returning service status and uptime. + +### Configuration + +The application can be configured using the following environment variables: + +| Variable | Default | Description | +|---------|-----------|--------------------------------------| +| `HOST` | `0.0.0.0` | Address to bind the HTTP server to | +| `PORT` | `5000` | Port to listen on | +| `DEBUG` | `False` | Enable Flask debug mode if `true` | + +Examples: + +```bash +HOST=127.0.0.1 PORT=8000 python app.py +DEBUG=true python app.py +``` + +### Docker + +The application can be run as a Docker container. + +**Build the image locally:** + +```bash +docker build -t devops-info-service . +``` + +**Run a container:** + +```bash +docker run -p 5000:5000 devops-info-service +``` + +Map the container port (5000) to a host port of your choice: `-p :5000`. +Override `PORT` or `HOST` with environment variables if needed. + +**Pull from Docker Hub:** + +```bash +docker pull jambulancia/devops-info-service +docker run -p 5000:5000 jambulancia/devops-info-service +``` + +### Testing + +```bash +cd app_python +python -m venv venv +source venv/bin/activate +pip install -r requirements-dev.txt +pytest tests/ -v +``` diff --git a/app_python/app.py b/app_python/app.py new file mode 100644 index 0000000000..75aec2b518 --- /dev/null +++ b/app_python/app.py @@ -0,0 +1,219 @@ +""" +DevOps Info Service +Main Flask application module. +""" + +import logging +import os +import platform +import socket +from datetime import datetime, timezone + +from flask import Flask, jsonify, request + +USE_JSON_LOGGING = os.getenv("LOG_FORMAT", "").lower() == "json" + +app = Flask(__name__) + +# Configuration +HOST = os.getenv("HOST", "0.0.0.0") +PORT = int(os.getenv("PORT", 5000)) +DEBUG = os.getenv("DEBUG", "False").lower() == "true" + +# Application start time (for uptime calculation) +START_TIME = datetime.now(timezone.utc) + +# Logging configuration +if USE_JSON_LOGGING: + from pythonjsonlogger import jsonlogger + + handler = logging.StreamHandler() + formatter = jsonlogger.JsonFormatter() + handler.setFormatter(formatter) + logging.root.handlers = [handler] + logging.root.setLevel(logging.INFO) + logger = logging.getLogger(__name__) +else: + logging.basicConfig( + level=logging.INFO, + format="%(asctime)s - %(name)s - %(levelname)s - %(message)s", + ) + logger = logging.getLogger(__name__) + +# Startup log +logger.info( + "Application starting", + extra={"host": HOST, "port": PORT, "debug": DEBUG} if USE_JSON_LOGGING else {}, +) + + +@app.before_request +def log_request(): + """Log incoming request (extra fields for JSON logging).""" + if USE_JSON_LOGGING: + logger.info( + "Request received", + extra={ + "method": request.method, + "path": request.path, + "client_ip": request.remote_addr or "unknown", + }, + ) + else: + logger.info("Request: %s %s from %s", request.method, request.path, request.remote_addr) + + +@app.after_request +def log_response(response): + """Log response status after request.""" + if USE_JSON_LOGGING: + logger.info( + "Response sent", + extra={ + "method": request.method, + "path": request.path, + "status_code": response.status_code, + "client_ip": request.remote_addr or "unknown", + }, + ) + return response + + +def get_system_info() -> dict: + """Collect basic system information.""" + return { + "hostname": socket.gethostname(), + "platform": platform.system(), + "platform_version": platform.version(), + "architecture": platform.machine(), + "cpu_count": os.cpu_count(), + "python_version": platform.python_version(), + } + + +def get_uptime() -> dict: + """Calculate uptime in seconds and human-readable form.""" + delta = datetime.now(timezone.utc) - START_TIME + seconds = int(delta.total_seconds()) + hours = seconds // 3600 + minutes = (seconds % 3600) // 60 + return { + "uptime_seconds": seconds, + "uptime_human": f"{hours} hours, {minutes} minutes", + } + + +def get_request_info() -> dict: + """Extract request-related information.""" + user_agent = request.headers.get("User-Agent") or request.headers.get( + "user-agent" + ) + return { + "client_ip": request.remote_addr, + "user_agent": user_agent, + "method": request.method, + "path": request.path, + } + + +@app.route("/", methods=["GET"]) +def index(): + """Main endpoint — service, system, runtime, and request information.""" + logger.info("Handling / request") + + uptime_info = get_uptime() + + response = { + "service": { + "name": "devops-info-service", + "version": "1.0.0", + "description": "DevOps course info service", + "framework": "Flask", + }, + "system": get_system_info(), + "runtime": { + "uptime_seconds": uptime_info["uptime_seconds"], + "uptime_human": uptime_info["uptime_human"], + "current_time": datetime.now(timezone.utc).isoformat(), + "timezone": "UTC", + }, + "request": get_request_info(), + "endpoints": [ + { + "path": "/", + "method": "GET", + "description": "Service information", + }, + { + "path": "/health", + "method": "GET", + "description": "Health check", + }, + ], + } + + logger.debug("Response payload for / endpoint generated") + return jsonify(response) + + +@app.route("/health", methods=["GET"]) +def health(): + """Health check endpoint.""" + logger.info("Handling /health request") + uptime_info = get_uptime() + response = { + "status": "healthy", + "timestamp": datetime.now(timezone.utc).isoformat(), + "uptime_seconds": uptime_info["uptime_seconds"], + } + return jsonify(response), 200 + + +@app.errorhandler(404) +def not_found(error): + """Handle 404 Not Found errors.""" + if USE_JSON_LOGGING: + logger.warning( + "404 Not Found", + extra={"method": request.method, "path": request.path}, + ) + else: + logger.warning("404 Not Found: %s %s", request.method, request.path) + return ( + jsonify( + { + "error": "Not Found", + "message": "Endpoint does not exist", + "path": request.path, + } + ), + 404, + ) + + +@app.errorhandler(500) +def internal_error(error): + """Handle 500 Internal Server Error.""" + logger.exception( + "500 Internal Server Error", + extra={"error": str(error)} if USE_JSON_LOGGING else {}, + ) + return ( + jsonify( + { + "error": "Internal Server Error", + "message": "An unexpected error occurred", + } + ), + 500, + ) + + +def main(): + """Application entrypoint.""" + logger.info("Starting DevOps Info Service on %s:%s (debug=%s)", HOST, PORT, DEBUG) + app.run(host=HOST, port=PORT, debug=DEBUG) + + +if __name__ == "__main__": + main() diff --git a/app_python/docs/LAB01.md b/app_python/docs/LAB01.md new file mode 100644 index 0000000000..33312a0883 --- /dev/null +++ b/app_python/docs/LAB01.md @@ -0,0 +1,150 @@ +## LAB01 — DevOps Info Service (Python) + +### 1. Framework Selection + +**Chosen Framework:** Flask + +| Criterion | Flask | FastAPI | Django | +|------------------------|----------------------------------|----------------------------------------|----------------------------------------| +| Learning curve | Very beginner-friendly | Moderate (type hints, async) | Steeper (full framework) | +| Use case fit | Simple APIs and microservices | High-performance APIs | Large, full-featured web apps | +| Ecosystem / extensions | Mature ecosystem, many examples | Great docs, built-in OpenAPI docs | Includes ORM, admin, auth, templates | +| Setup complexity | Minimal | Minimal | Higher (project + apps structure) | +| For this lab | Ideal for quick REST services | Slightly more complex than necessary | Overkill | + +**Why Flask?** + +- The lab only needs two simple HTTP endpoints and JSON responses. +- Flask is lightweight, easy to understand, and perfect for a small service that will grow over time. +- There is a lot of learning material and community support, which is helpful for beginners. + +### 2. Best Practices Applied + +- **Clean Code Organization** + - Clear function names such as `get_system_info()`, `get_uptime()`, and `get_request_info()`. + - Configuration values (`HOST`, `PORT`, `DEBUG`) are defined at the top of `app.py`. + - A `main()` function is used as the entrypoint to keep `if __name__ == "__main__":` minimal. + +- **PEP 8 Compliance** + - Imports are grouped (standard library first, then third-party). + - Snake_case is used for function and variable names. + - Line lengths and spacing follow PEP 8 conventions. + +- **Error Handling** + - Custom handlers for `404` and `500` errors return JSON responses: + - `404` includes an error message and the invalid path. + - `500` returns a generic error message without leaking internals. + +- **Logging** + - Configured via `logging.basicConfig` with timestamp, logger name, level, and message. + - Logs when the application starts and when requests to `/` and `/health` are handled. + - Errors and 500s are logged with stack traces for easier debugging. + +### 3. API Documentation + +#### `GET /` + +- **Description:** Returns service, system, runtime, and request information, plus a list of available endpoints. +- **Example Request:** + +```bash +curl http://localhost:5000/ +``` + +- **Example Response (truncated):** + +```json +{ + "service": { + "name": "devops-info-service", + "version": "1.0.0", + "description": "DevOps course info service", + "framework": "Flask" + }, + "system": { + "hostname": "my-laptop", + "platform": "Linux", + "platform_version": "Ubuntu 24.04", + "architecture": "x86_64", + "cpu_count": 8, + "python_version": "3.11.0" + }, + "runtime": { + "uptime_seconds": 12, + "uptime_human": "0 hours, 0 minutes", + "current_time": "2026-01-27T14:30:00.000Z", + "timezone": "UTC" + }, + "request": { + "client_ip": "127.0.0.1", + "user_agent": "curl/7.81.0", + "method": "GET", + "path": "/" + }, + "endpoints": [ + { "path": "/", "method": "GET", "description": "Service information" }, + { "path": "/health", "method": "GET", "description": "Health check" } + ] +} +``` + +#### `GET /health` + +- **Description:** Simple health check used for readiness/liveness probes. +- **Example Request:** + +```bash +curl http://localhost:5000/health +``` + +- **Example Response:** + +```json +{ + "status": "healthy", + "timestamp": "2026-01-27T14:30:00.000Z", + "uptime_seconds": 42 +} +``` + +### 4. Testing Evidence + +**Manual Testing Commands** + +From the `app_python` directory: + +```bash +python app.py + +# In another terminal +curl http://localhost:5000/ | jq +curl http://localhost:5000/health | jq +``` + +- `curl` fetches the JSON responses. +- `jq` pretty-prints the JSON output for easier reading. + +**Screenshots (to be added by you):** + +Place the following screenshots in `app_python/docs/screenshots/`: + +- `01-main-endpoint.png` — Browser or terminal showing the full JSON from `GET /`. +- `02-health-check.png` — Response from `GET /health`. +- `03-formatted-output.png` — Pretty-printed JSON output (e.g., using `jq` or browser dev tools). + +### 5. Challenges & Solutions + +- **Challenge:** Calculating and formatting uptime correctly. + - **Solution:** Store a global `START_TIME` when the app starts and compute the time difference on each request, returning both seconds and a human-readable `"{hours} hours, {minutes} minutes"` string. + +- **Challenge:** Making the app configurable without changing code. + - **Solution:** Read `HOST`, `PORT`, and `DEBUG` from environment variables with sensible defaults, so the same code can run in different environments. + +### 6. GitHub Community + +- **Why starring repositories matters:** + Starring repositories is a lightweight way to bookmark useful projects and signal appreciation to maintainers. A higher star count helps good projects become more visible, attract contributors, and build trust in the open-source community. + +- **How following developers helps:** + Following professors, TAs, and classmates makes it easier to discover new projects, see how others solve problems, and stay connected with your learning community. Over time this builds a professional network and exposes you to real-world development practices. + diff --git a/app_python/docs/LAB02.md b/app_python/docs/LAB02.md new file mode 100644 index 0000000000..0fb0c3348c --- /dev/null +++ b/app_python/docs/LAB02.md @@ -0,0 +1,136 @@ +## LAB02 — Docker Containerization (Python) + +### 1. Docker Best Practices Applied + +#### Non-root user +- **What:** Created a dedicated `appuser` in `appgroup` and switched to it with `USER appuser`. +- **Why:** Running as root inside a container is a security risk. If the app is compromised, an attacker would have root access. Non-root users limit the blast radius and follow the principle of least privilege. + +```dockerfile +RUN groupadd -r appgroup && useradd -r -g appgroup appuser +# ... copy files, chown ... +USER appuser +``` + +#### Specific base image version +- **What:** Use `python:3.13-slim` instead of `python:3` or `python:latest`. +- **Why:** Pinned versions ensure reproducible builds and avoid surprises when base images change. `slim` is smaller than the full image (no build tools, fewer packages) while still being easy to work with. + +#### Layer ordering for caching +- **What:** Copy `requirements.txt` first, run `pip install`, then copy `app.py`. +- **Why:** Dependencies change less often than application code. Docker caches each layer; if only `app.py` changes, the `pip install` layer is reused, making rebuilds faster. + +```dockerfile +COPY requirements.txt . +RUN pip install --no-cache-dir -r requirements.txt +COPY app.py . +``` + +#### Only copy necessary files +- **What:** Copy only `requirements.txt` and `app.py`; use `.dockerignore` to exclude the rest. +- **Why:** Smaller build context speeds up uploads to the Docker daemon. Fewer files in the image reduce attack surface and image size. + +#### .dockerignore +- **What:** Exclude `__pycache__`, `venv`, `.git`, `docs`, `tests`, IDE configs, etc. +- **Why:** These files are not needed at runtime and would bloat the build context and potentially the image. Excluding them speeds up builds and keeps images lean. + +#### PYTHONDONTWRITEBYTECODE and PYTHONUNBUFFERED +- **What:** Set `ENV PYTHONDONTWRITEBYTECODE=1 PYTHONUNBUFFERED=1`. +- **Why:** Avoid writing `.pyc` files and buffer stdout/stderr so logs appear immediately in `docker logs`. + +--- + +### 2. Image Information & Decisions + +| Decision | Choice | Justification | +|----------|--------|---------------| +| Base image | `python:3.13-slim` | Latest stable Python, slim variant for smaller size without sacrificing compatibility | +| Final image size | ~150–180 MB | Typical for python:3.13-slim + Flask; acceptable for a dev/lab service | +| Layer structure | Base → deps → user → app → USER | Dependencies first for cache, user setup before app copy, USER last | + +**Optimization choices:** +- `--no-cache-dir` with pip to avoid keeping package cache in the image +- Minimal COPYs (only `requirements.txt` and `app.py`) +- No build tools or unnecessary packages + +--- + +### 3. Build & Run Process + +**Build:** + +```bash +cd app_python +docker build -t devops-info-service . +``` + +![Docker build output](screenshots/docker_build.png) + +**Run container:** + +```bash +docker run -d -p 5000:5000 --name devops-app devops-info-service +``` + +![Docker run output](screenshots/docker_run.png) + +**Test endpoints:** + +```bash +curl http://localhost:5000/ +curl http://localhost:5000/health +``` + +**Docker Hub:** + +![Docker Hub repository](screenshots/dockerhub.png) + +**Pull and run from Docker Hub** (verifies image is publicly accessible): + +```bash +docker pull jambulancia/devops-info-service:latest +docker run -d -p 5000:5000 --name devops-app jambulancia/devops-info-service:latest +curl http://localhost:5000/health +``` + +![Pull and run from Docker Hub](screenshots/run_dockerhub.png) + +**Repository URL:** https://hub.docker.com/r/jambulancia/devops-info-service + +**Tagging strategy:** Image tagged as `jambulancia/devops-info-service:latest` — `latest` for the current stable build; version tags (e.g. `1.0.0`) can be added later for releases. + +--- + +### 4. Technical Analysis + +**Why does the Dockerfile work this way?** +- The app binds to `0.0.0.0` (all interfaces) by default, so it is reachable from outside the container. +- Port 5000 is exposed; `-p 5000:5000` maps host 5000 to container 5000. +- The `USER` directive ensures the process runs as `appuser`, not root. + +**What if layer order changed?** +- If we copied `app.py` before `pip install`, any code change would invalidate the cache for `pip install`, forcing a full reinstall on every build. +- Putting `USER` before `COPY` would cause permission errors unless we copy as root and then chown (which we do). + +**Security considerations:** +- Non-root user reduces impact of container escape or app compromise. +- Minimal base image and fewer files shrink the attack surface. +- No secrets or credentials in the image. + +**How does .dockerignore improve the build?** +- Reduces the amount of data sent to the Docker daemon during `docker build`. +- Avoids including `.git`, `venv`, or other large/unnecessary directories. +- Faster builds and cleaner images. + +--- + +### 5. Challenges & Solutions + +**Challenge:** Ensuring the app listens on `0.0.0.0` inside the container. +- **Solution:** The app already uses `HOST = os.getenv("HOST", "0.0.0.0")`, so it binds to all interfaces by default. No change needed. + +**Challenge:** Non-root user and file ownership. +- **Solution:** Create the user, copy files as root, run `chown -R appuser:appgroup /app`, then `USER appuser` so the app can read its files. + +**Challenge:** Keeping the image small. +- **Solution:** Use `python:3.13-slim`, `--no-cache-dir` for pip, and `.dockerignore` to exclude dev artifacts. diff --git a/app_python/docs/LAB03.md b/app_python/docs/LAB03.md new file mode 100644 index 0000000000..288378bd76 --- /dev/null +++ b/app_python/docs/LAB03.md @@ -0,0 +1,111 @@ +# LAB03 — Continuous Integration (CI/CD) + +## 1. Overview + +### Testing Framework + +**pytest** — Chosen for its simple syntax, strong fixture support, and wide adoption. It integrates well with Flask's test client and supports clear assertions. + +### Test Coverage + +- **GET /** — Status 200, JSON structure, required fields (service, system, runtime, request, endpoints), data types +- **GET /health** — Status 200, `status=healthy`, `uptime_seconds`, `timestamp` +- **404** — Non-existent paths return 404 with error structure +- **Request context** — User-Agent and path reflected in response + +### CI Workflow Triggers + +- **Push** to `master`, `lab02`, `lab03` +- **Pull request** to `master`, `lab02`, `lab03` +- **Path filter:** Only when `app_python/**` or `.github/workflows/python-ci.yml` changes + +### Versioning Strategy + +**CalVer (Calendar Versioning)** — Format `YYYY.MM.DD` (e.g. `2026.02.04`). Chosen because this is a service deployed continuously rather than a library with breaking- change semantics. CalVer gives clear, date-based versions without manual tagging. + +--- + +## 2. Workflow Evidence + +| Item | Link / Evidence | +|------|-----------------| +| Successful workflow run | [GitHub Actions](https://github.com/abdughafforzoda/DevOps-Core-Course/actions/workflows/python-ci.yml) | +| Tests passing locally | See terminal output below | +| Docker image on Docker Hub | [jambulancia/devops-info-service](https://hub.docker.com/r/jambulancia/devops-info-service) | +| Status badge | In `app_python/README.md` | + +**Tests passing locally:** + +``` +============================= test session starts ============================== +platform linux -- Python 3.13.3, pytest-9.0.2 +collected 13 items + +tests/test_app.py::test_index_returns_200 PASSED +tests/test_app.py::test_index_returns_json PASSED +tests/test_app.py::test_index_service_structure PASSED +tests/test_app.py::test_index_system_structure PASSED +tests/test_app.py::test_index_runtime_structure PASSED +tests/test_app.py::test_index_request_structure PASSED +tests/test_app.py::test_index_endpoints_list PASSED +tests/test_app.py::test_health_returns_200 PASSED +tests/test_app.py::test_health_returns_json PASSED +tests/test_app.py::test_health_structure PASSED +tests/test_app.py::test_404_nonexistent_endpoint PASSED +tests/test_app.py::test_404_wrong_method PASSED +tests/test_app.py::test_index_request_has_client_ip PASSED + +============================== 13 passed in 0.06s ============================== +``` + +--- + +## 3. Best Practices Implemented + +- **Dependency caching** — `actions/setup-python` with `cache: 'pip'` caches pip packages; speeds up jobs by ~30–60s on cache hit. +- **Concurrency** — `concurrency` cancels outdated workflow runs when new commits are pushed. +- **Path filters** — CI runs only when Python app files change, reducing unnecessary runs. +- **Job dependencies** — Docker job runs only after tests pass (`needs: test`). +- **Conditional Docker push** — Images pushed only on `push` (not on `pull_request`). +- **Snyk** — Vulnerability scan with `continue-on-error: true` so missing `SNYK_TOKEN` does not fail CI. Add `SNYK_TOKEN` for full scanning. +- **Docker layer caching** — `cache-from/cache-to: type=gha` reuses build layers between runs. + +--- + +## 4. Key Decisions + +| Decision | Choice | Rationale | +|----------|--------|-----------| +| **Versioning** | CalVer | Continuous deployment; date-based releases without manual version bumps. | +| **Docker tags** | `YYYY.MM.DD` + `latest` | E.g. `jambulancia/devops-info-service:2026.02.04` and `:latest`. | +| **Triggers** | Push + PR to master, lab02, lab03 | Validate changes before and after merge; runs on relevant branches. | +| **Test coverage** | All endpoints, structure, types | Ensures JSON shape and required fields; omits 500 handler due to needing forced failure. | +| **Linter** | Ruff | Fast, modern linter with good defaults. | + +--- + +## 6. Bonus: Multi-App CI with Path Filters + +A separate **Go CI** workflow (`.github/workflows/go-ci.yml`) runs when `app_go/**` changes. Both workflows use path filters so that: + +- Changes to `app_python/` → only Python CI runs +- Changes to `app_go/` → only Go CI runs +- Changes to both → both run in parallel +- Changes to `docs/` or `labs/` → neither runs + +**Benefits:** Fewer unnecessary runs, faster feedback, and lower Actions usage. + +--- + +## 7. Setup Required + +Before the workflow runs correctly: + +1. **Docker Hub** — Add secrets in GitHub: `Settings → Secrets and variables → Actions`: + - `DOCKERHUB_USERNAME`: your Docker Hub username + - `DOCKERHUB_TOKEN`: Docker Hub access token (create at hub.docker.com) + +2. **Snyk (optional)** — For security scanning: + - Create account at snyk.io + - Add `SNYK_TOKEN` as a GitHub secret + - Without it, the Snyk step is skipped (`continue-on-error: true`) diff --git a/app_python/docs/screenshots/01-main-endpoint.png b/app_python/docs/screenshots/01-main-endpoint.png new file mode 100644 index 0000000000..8d04514dcd Binary files /dev/null and b/app_python/docs/screenshots/01-main-endpoint.png differ diff --git a/app_python/docs/screenshots/02-health-check.png b/app_python/docs/screenshots/02-health-check.png new file mode 100644 index 0000000000..1147af9587 Binary files /dev/null and b/app_python/docs/screenshots/02-health-check.png differ diff --git a/app_python/docs/screenshots/03-formatted-output.png b/app_python/docs/screenshots/03-formatted-output.png new file mode 100644 index 0000000000..65dc6537b8 Binary files /dev/null and b/app_python/docs/screenshots/03-formatted-output.png differ diff --git a/app_python/docs/screenshots/docker_build.png b/app_python/docs/screenshots/docker_build.png new file mode 100644 index 0000000000..14b715f92a Binary files /dev/null and b/app_python/docs/screenshots/docker_build.png differ diff --git a/app_python/docs/screenshots/docker_run.png b/app_python/docs/screenshots/docker_run.png new file mode 100644 index 0000000000..c110c29eb8 Binary files /dev/null and b/app_python/docs/screenshots/docker_run.png differ diff --git a/app_python/docs/screenshots/dockerhub.png b/app_python/docs/screenshots/dockerhub.png new file mode 100644 index 0000000000..bdd833dfdf Binary files /dev/null and b/app_python/docs/screenshots/dockerhub.png differ diff --git a/app_python/docs/screenshots/run_dockerhub.png b/app_python/docs/screenshots/run_dockerhub.png new file mode 100644 index 0000000000..e55fb6e230 Binary files /dev/null and b/app_python/docs/screenshots/run_dockerhub.png differ diff --git a/app_python/pytest.ini b/app_python/pytest.ini new file mode 100644 index 0000000000..2476a922fb --- /dev/null +++ b/app_python/pytest.ini @@ -0,0 +1,5 @@ +[pytest] +testpaths = tests +python_files = test_*.py +python_functions = test_* +addopts = -v diff --git a/app_python/requirements-dev.txt b/app_python/requirements-dev.txt new file mode 100644 index 0000000000..597f5f362d --- /dev/null +++ b/app_python/requirements-dev.txt @@ -0,0 +1,4 @@ +-r requirements.txt +pytest>=8.0.0 +pytest-cov>=4.1.0 +ruff>=0.8.0 diff --git a/app_python/requirements.txt b/app_python/requirements.txt new file mode 100644 index 0000000000..e35baa8aed --- /dev/null +++ b/app_python/requirements.txt @@ -0,0 +1,3 @@ +Flask==3.1.0 +python-json-logger==2.0.7 + diff --git a/app_python/tests/__init__.py b/app_python/tests/__init__.py new file mode 100644 index 0000000000..95e046e0e0 --- /dev/null +++ b/app_python/tests/__init__.py @@ -0,0 +1,6 @@ +""" +Test package for the DevOps Info Service. + +Unit tests for this application will be added in Lab 3. +""" + diff --git a/app_python/tests/conftest.py b/app_python/tests/conftest.py new file mode 100644 index 0000000000..34ad41c216 --- /dev/null +++ b/app_python/tests/conftest.py @@ -0,0 +1,13 @@ +"""Pytest fixtures for DevOps Info Service tests.""" + +import pytest + +from app import app + + +@pytest.fixture +def client(): + """Create a Flask test client.""" + app.config["TESTING"] = True + with app.test_client() as c: + yield c diff --git a/app_python/tests/test_app.py b/app_python/tests/test_app.py new file mode 100644 index 0000000000..1fcd80ba24 --- /dev/null +++ b/app_python/tests/test_app.py @@ -0,0 +1,129 @@ +"""Unit tests for DevOps Info Service endpoints.""" + + +def test_index_returns_200(client): + """GET / returns 200 OK.""" + response = client.get("/") + assert response.status_code == 200 + + +def test_index_returns_json(client): + """GET / returns valid JSON.""" + response = client.get("/") + assert response.content_type == "application/json" + data = response.get_json() + assert data is not None + + +def test_index_service_structure(client): + """GET / includes required service metadata.""" + response = client.get("/") + data = response.get_json() + assert "service" in data + service = data["service"] + assert service["name"] == "devops-info-service" + assert service["version"] == "1.0.0" + assert service["description"] == "DevOps course info service" + assert service["framework"] == "Flask" + + +def test_index_system_structure(client): + """GET / includes system info with required fields.""" + response = client.get("/") + data = response.get_json() + assert "system" in data + system = data["system"] + assert "hostname" in system + assert "platform" in system + assert "platform_version" in system + assert "architecture" in system + assert "cpu_count" in system + assert "python_version" in system + assert isinstance(system["cpu_count"], (int, type(None))) + + +def test_index_runtime_structure(client): + """GET / includes runtime info with required fields.""" + response = client.get("/") + data = response.get_json() + assert "runtime" in data + runtime = data["runtime"] + assert "uptime_seconds" in runtime + assert "uptime_human" in runtime + assert "current_time" in runtime + assert runtime["timezone"] == "UTC" + assert isinstance(runtime["uptime_seconds"], int) + + +def test_index_request_structure(client): + """GET / includes request info from the client.""" + response = client.get("/", headers={"User-Agent": "test-agent/1.0"}) + data = response.get_json() + assert "request" in data + req = data["request"] + assert "client_ip" in req + assert "user_agent" in req + assert req["user_agent"] == "test-agent/1.0" + assert req["method"] == "GET" + assert req["path"] == "/" + + +def test_index_endpoints_list(client): + """GET / includes list of available endpoints.""" + response = client.get("/") + data = response.get_json() + assert "endpoints" in data + endpoints = data["endpoints"] + assert len(endpoints) >= 2 + paths = [e["path"] for e in endpoints] + assert "/" in paths + assert "/health" in paths + + +def test_health_returns_200(client): + """GET /health returns 200 OK.""" + response = client.get("/health") + assert response.status_code == 200 + + +def test_health_returns_json(client): + """GET /health returns valid JSON.""" + response = client.get("/health") + assert response.content_type == "application/json" + data = response.get_json() + assert data is not None + + +def test_health_structure(client): + """GET /health includes status, timestamp, uptime_seconds.""" + response = client.get("/health") + data = response.get_json() + assert data["status"] == "healthy" + assert "timestamp" in data + assert "uptime_seconds" in data + assert isinstance(data["uptime_seconds"], int) + + +def test_404_nonexistent_endpoint(client): + """GET /nonexistent returns 404 with error structure.""" + response = client.get("/nonexistent") + assert response.status_code == 404 + data = response.get_json() + assert "error" in data + assert data["error"] == "Not Found" + assert "path" in data + assert data["path"] == "/nonexistent" + + +def test_404_wrong_method(client): + """POST / returns 405 or 404 (method not allowed or not found).""" + response = client.post("/") + assert response.status_code in (404, 405) + + +def test_index_request_has_client_ip(client): + """Request info includes client_ip field.""" + response = client.get("/") + data = response.get_json() + assert "request" in data + assert "client_ip" in data["request"] diff --git a/monitoring/.env.example b/monitoring/.env.example new file mode 100644 index 0000000000..45a0ac5803 --- /dev/null +++ b/monitoring/.env.example @@ -0,0 +1,3 @@ +# Production Grafana config. Copy to .env and customize. Do not commit .env. +GF_AUTH_ANONYMOUS_ENABLED=false +GF_SECURITY_ADMIN_PASSWORD=changeme diff --git a/monitoring/docker-compose.yml b/monitoring/docker-compose.yml new file mode 100644 index 0000000000..c07303c5ac --- /dev/null +++ b/monitoring/docker-compose.yml @@ -0,0 +1,127 @@ +services: + loki: + image: grafana/loki:3.0.0 + container_name: loki + ports: + - "3100:3100" + volumes: + - ./loki/config.yml:/etc/loki/config.yml:ro + - loki-data:/loki + command: -config.file=/etc/loki/config.yml + networks: + - logging + deploy: + resources: + limits: + cpus: "1.0" + memory: 1G + reservations: + cpus: "0.25" + memory: 256M + healthcheck: + test: ["CMD-SHELL", "wget --no-verbose --tries=1 --spider http://localhost:3100/ready || exit 1"] + interval: 10s + timeout: 5s + retries: 5 + start_period: 10s + labels: + logging: "promtail" + app: "loki" + + promtail: + image: grafana/promtail:3.0.0 + container_name: promtail + ports: + - "9080:9080" + volumes: + - ./promtail/config.yml:/etc/promtail/config.yml:ro + - /var/lib/docker/containers:/var/lib/docker/containers:ro + - /var/run/docker.sock:/var/run/docker.sock:ro + command: -config.file=/etc/promtail/config.yml + networks: + - logging + depends_on: + loki: + condition: service_healthy + deploy: + resources: + limits: + cpus: "0.5" + memory: 512M + reservations: + cpus: "0.1" + memory: 128M + labels: + logging: "promtail" + app: "promtail" + + grafana: + image: grafana/grafana:12.3.1 + container_name: grafana + ports: + - "3000:3000" + environment: + # Dev: anonymous enabled. For production, set GF_AUTH_ANONYMOUS_ENABLED=false in .env + GF_AUTH_ANONYMOUS_ENABLED: "${GF_AUTH_ANONYMOUS_ENABLED:-true}" + GF_AUTH_ANONYMOUS_ORG_ROLE: Admin + GF_SECURITY_ALLOW_EMBEDDING: "true" + GF_SERVER_ROOT_URL: "http://localhost:3000" + GF_SECURITY_ADMIN_PASSWORD: "${GF_SECURITY_ADMIN_PASSWORD:-admin}" + volumes: + - grafana-data:/var/lib/grafana + networks: + - logging + depends_on: + loki: + condition: service_healthy + deploy: + resources: + limits: + cpus: "0.5" + memory: 512M + reservations: + cpus: "0.1" + memory: 128M + healthcheck: + test: ["CMD-SHELL", "wget --no-verbose --tries=1 --spider http://localhost:3000/api/health || exit 1"] + interval: 10s + timeout: 5s + retries: 5 + start_period: 30s + labels: + logging: "promtail" + app: "grafana" + + app-python: + image: jambulancia/devops-info-service:latest + container_name: devops-python + ports: + - "8000:5000" + # Uncomment for direct Loki push (requires: docker plugin install grafana/loki-docker-driver:3.6.0 --alias loki --grant-all-permissions) + # logging: + # driver: loki + # options: + # loki-url: "http://loki:3100/loki/api/v1/push" + # loki-external-labels: "job=docker,app=devops-python" + environment: + PORT: "5000" + HOST: "0.0.0.0" + LOG_FORMAT: "json" + networks: + - logging + deploy: + resources: + limits: + cpus: "0.5" + memory: 256M + labels: + logging: "promtail" + app: "devops-python" + +volumes: + loki-data: + grafana-data: + +networks: + logging: + driver: bridge diff --git a/monitoring/docs/LAB07.md b/monitoring/docs/LAB07.md new file mode 100644 index 0000000000..563ddd0096 --- /dev/null +++ b/monitoring/docs/LAB07.md @@ -0,0 +1,228 @@ +# Lab 7 — Observability & Logging with Loki Stack + + +## 1. Architecture + +``` +┌─────────────────┐ ┌─────────────────┐ ┌─────────────────┐ +│ app-python │ │ Loki │ │ Grafana │ +│ (Flask) │ │ (Log Storage) │ │ (Visualization)│ +│ Port 8000 │ │ Port 3100 │ │ Port 3000 │ +└────────┬────────┘ └────────▲────────┘ └────────▲────────┘ + │ │ │ + │ JSON logs │ push │ query + │ (stdout) │ │ + ▼ │ │ +┌─────────────────┐ ┌────────┴────────┐ ┌───────┴────────┐ +│ Docker │ │ Promtail │ │ Loki Data │ +│ (json-file │────▶│ (Log Collector)│────▶│ Source │ +│ log driver) │ │ Port 9080 │ │ (preconfigured)│ +└─────────────────┘ └─────────────────┘ └────────────────┘ + │ │ + │ /var/lib/docker/ │ Docker SD + │ containers │ + filters + └───────────────────────┘ + +All services run on the `logging` bridge network. +Promtail discovers containers via Docker socket, reads log files, and pushes to Loki. +``` + +**Component roles:** +- **Loki:** Stores logs with TSDB index; 7-day retention; compactor cleans old data +- **Promtail:** Discovers containers (label `logging=promtail`), reads Docker log files, pushes to Loki +- **Grafana:** Queries Loki via LogQL, dashboards, Explore + +--- + +## 2. Setup Guide + +### Prerequisites +- Docker and Docker Compose v2 +- Lab 1 Python app built as `jambulancia/devops-info-service:latest` + +### Deploy + +```bash +cd monitoring +docker compose up -d +docker compose ps +``` + +### Verify + +```bash +# Loki readiness +curl http://localhost:3100/ready + +# Promtail targets (log discovery) +curl http://localhost:9080/targets + +# Grafana +open http://localhost:3000 +``` + +### Configure Loki Data Source in Grafana + +1. **Connections** → **Data sources** → **Add data source** → **Loki** +2. URL: `http://loki:3100` +3. **Save & Test** + +### Generate Logs + +```bash +for i in {1..20}; do curl http://localhost:8000/; done +for i in {1..20}; do curl http://localhost:8000/health; done +``` + +### Alternative: Loki Docker Logging Driver + +If Promtail Docker SD does not push logs in your environment, you can use the Loki Docker logging driver for the app: + +```bash +docker plugin install grafana/loki-docker-driver:3.6.0 --alias loki --grant-all-permissions +``` + +Then add to `app-python` in docker-compose: + +```yaml + logging: + driver: loki + options: + loki-url: "http://loki:3100/loki/api/v1/push" + loki-external-labels: "job=docker,app=devops-python" +``` + +--- + +## 3. Configuration + +### Loki (`loki/config.yml`) + +- **Schema:** v13 with TSDB and filesystem storage +- **Retention:** 168h (7 days) via `limits_config.retention_period` +- **Compactor:** Enabled to delete data beyond retention + +```yaml +limits_config: + retention_period: 168h + +compactor: + retention_enabled: true +``` + +### Promtail (`promtail/config.yml`) + +- **Docker SD:** Discovers containers with label `logging=promtail` +- **`__path__` relabel:** Points to `/var/lib/docker/containers/${id}/*.log` so Promtail reads Docker log files +- **Labels:** `container` (from name), `service` (from compose service label) +- **Pipeline:** `docker: {}` parses Docker JSON log wrapper + +```yaml +relabel_configs: + - source_labels: ['__meta_docker_container_id'] + regex: '(.+)' + target_label: __path__ + replacement: /var/lib/docker/containers/${1}/*.log + - source_labels: ['__meta_docker_container_name'] + regex: '/(.*)' + target_label: 'container' + - source_labels: ['__meta_docker_container_label_com_docker_compose_service'] + target_label: 'service' +``` + +--- + +## 4. Application Logging + +The Python app uses `python-json-logger` when `LOG_FORMAT=json` is set. + +- **Format:** `{"timestamp": "...", "level": "...", "message": "...", "method": "...", "path": "...", ...}` +- **Events logged:** Startup, request received, response sent, 404, 500 +- **Context:** method, path, status_code, client_ip + +```python +# app_python/app.py +from pythonjsonlogger import jsonlogger + +if USE_JSON_LOGGING: + formatter = jsonlogger.JsonFormatter() + handler.setFormatter(formatter) +``` + +Docker Compose sets `LOG_FORMAT: "json"` for the app service. + +--- + +## 5. Dashboard + +Create a dashboard with 4 panels: + +| Panel | Type | LogQL Query | +|--------------------|-------------|-----------------------------------------------------------------------------| +| Logs Table | Logs | `{app=~"devops-.*"}` | +| Request Rate | Time series | `sum by (app) (rate({app=~"devops-.*"} [1m]))` | +| Error Logs | Logs | `{app=~"devops-.*"} \| json \| level="ERROR"` | +| Log Level Distribution | Stat/Pie | `sum by (level) (count_over_time({app=~"devops-.*"} \| json [5m]))` | + +**How to create:** +1. **Dashboard** → **New** → **New Dashboard** → **Add visualization** +2. Select **Loki** data source +3. Enter LogQL and choose visualization type + +--- + +## 6. Production Config + +- **Resource limits:** All services have `deploy.resources.limits` (CPU, memory) +- **Health checks:** Loki (`/ready`), Grafana (`/api/health`) +- **Grafana security:** Disable anonymous auth for production: + - `GF_AUTH_ANONYMOUS_ENABLED: "false"` + - `GF_SECURITY_ADMIN_PASSWORD` from `.env` (do not commit) +- **Secrets:** Use `.env` for Grafana admin password; add to `.gitignore` + +--- + +## 7. Testing + +```bash +# Full stack health +cd monitoring +docker compose ps + +# Loki +curl -s http://localhost:3100/ready + +# Promtail targets +curl -s http://localhost:9080/targets | head -50 + +# Generate traffic +for i in {1..10}; do curl -s http://localhost:8000/ > /dev/null; done +for i in {1..10}; do curl -s http://localhost:8000/health > /dev/null; done +``` + +**Grafana Explore:** Run `{job="docker"}` or `{app="devops-python"}` to confirm logs appear. + +--- + +## 8. Challenges & Solutions + +| Challenge | Solution | +|------------------------------------|--------------------------------------------------------------------------| +| Promtail Docker SD not pushing logs| Ensure Docker socket + `/var/lib/docker/containers` are mounted. Try `curl localhost:9080/targets`. Alternative: use [Loki Docker logging driver](https://grafana.com/docs/loki/latest/send-data/docker-driver/) | +| Too many containers discovered | Use `filters: - name: label values: ["logging=promtail"]` in Docker SD | +| JSON parsing in LogQL | Use `\| json` pipeline stage; filter with `level="ERROR"` | +| Label vs service name | Use `__meta_docker_container_label_com_docker_compose_service` for app | +| Loki compactor config error | Add `delete_request_store: filesystem` when `retention_enabled: true` | + +--- + +## Evidence Checklist + +- [x] Loki, Promtail, Grafana running via Docker Compose +- [x] Loki data source in Grafana +- [x] Python app with JSON logging +- [x] Logs visible in Grafana from all labeled containers +- [x] Dashboard with 4 panels +- [x] LogQL queries for streams, errors, rates, levels +- [x] Resource limits and health checks +- [x] LAB07.md with setup and config notes diff --git a/monitoring/loki/config.yml b/monitoring/loki/config.yml new file mode 100644 index 0000000000..2d54cfb545 --- /dev/null +++ b/monitoring/loki/config.yml @@ -0,0 +1,50 @@ +auth_enabled: false + +server: + http_listen_port: 3100 + grpc_listen_port: 9096 + log_level: info + +common: + path_prefix: /loki + replication_factor: 1 + ring: + kvstore: + store: inmemory + +query_range: + results_cache: + cache: + embedded_cache: + enabled: true + max_size_mb: 100 + +limits_config: + retention_period: 168h # 7 days + +compactor: + working_directory: /loki/compactor + compaction_interval: 10m + retention_enabled: true + delete_request_store: filesystem + +schema_config: + configs: + - from: 2020-10-24 + store: tsdb + object_store: filesystem + schema: v13 + index: + prefix: index_ + period: 24h + +storage_config: + filesystem: + directory: /loki/chunks + +ruler: + alertmanager_url: http://localhost:9093 + storage: + type: local + local: + directory: /loki/rules diff --git a/monitoring/promtail/config.yml b/monitoring/promtail/config.yml new file mode 100644 index 0000000000..b8e92a3604 --- /dev/null +++ b/monitoring/promtail/config.yml @@ -0,0 +1,30 @@ +server: + http_listen_port: 9080 + grpc_listen_port: 0 + +positions: + filename: /tmp/positions.yaml + +clients: + - url: http://loki:3100/loki/api/v1/push + +scrape_configs: + - job_name: docker + docker_sd_configs: + - host: unix:///var/run/docker.sock + refresh_interval: 5s + filters: + - name: label + values: ["logging=promtail"] + relabel_configs: + - source_labels: ['__meta_docker_container_name'] + regex: '/(.*)' + target_label: 'container' + - source_labels: ['__meta_docker_container_label_com_docker_compose_service'] + target_label: 'service' + regex: '(.+)' + - source_labels: ['__meta_docker_container_label_app'] + target_label: 'app' + regex: '(.+)' + pipeline_stages: + - docker: {} diff --git a/pulumi/.gitignore b/pulumi/.gitignore new file mode 100644 index 0000000000..439e8a21a6 --- /dev/null +++ b/pulumi/.gitignore @@ -0,0 +1,17 @@ +# Pulumi +Pulumi.*.yaml +!Pulumi.yaml + +# Python +__pycache__/ +*.py[cod] +.venv/ +venv/ +*.egg-info/ + +# IDE +.idea/ +.vscode/ + +# OS +.DS_Store diff --git a/pulumi/Pulumi.yaml b/pulumi/Pulumi.yaml new file mode 100644 index 0000000000..1ed0e01dc6 --- /dev/null +++ b/pulumi/Pulumi.yaml @@ -0,0 +1,3 @@ +name: devops-lab04 +runtime: python +description: Lab 4 - VM on AWS (same as Terraform) diff --git a/pulumi/README.md b/pulumi/README.md new file mode 100644 index 0000000000..3aa1b2ac1e --- /dev/null +++ b/pulumi/README.md @@ -0,0 +1,46 @@ +# Pulumi — Lab 4 IaC + +Same infrastructure as Terraform: one EC2 VM on AWS (free tier) with SSH, HTTP, and port 5000. + +## Prerequisites + +- [Pulumi CLI](https://www.pulumi.com/docs/install/) +- Python 3.9+ +- AWS credentials (env or `~/.aws/credentials`) +- SSH public key at `~/.ssh/id_rsa.pub` (or set `ssh_public_key_path` in config) + +## Setup + +```bash +cd pulumi +python -m venv venv +source venv/bin/activate # or venv\Scripts\activate on Windows +pip install -r requirements.txt +``` + +## Config (optional) + +```bash +pulumi config set aws:region us-east-1 +pulumi config set project_name devops-lab04 +pulumi config set ssh_public_key_path ~/.ssh/id_rsa.pub +pulumi config set allowed_ssh_cidr "YOUR_IP/32" +``` + +## Deploy + +```bash +pulumi preview +pulumi up +pulumi stack output +``` + +## Cleanup + +```bash +pulumi destroy +``` + +## Note + +Destroy Terraform resources before running Pulumi (or use a different project_name/region) to avoid name conflicts (e.g. key pair name). diff --git a/pulumi/__main__.py b/pulumi/__main__.py new file mode 100644 index 0000000000..ecac7b308c --- /dev/null +++ b/pulumi/__main__.py @@ -0,0 +1,64 @@ +"""Lab 4 - Create VM on AWS with Pulumi (same as Terraform).""" +import os +import pulumi +import pulumi_aws as aws + +config = pulumi.Config() +project_name = config.get("project_name") or "devops-lab04" +instance_type = config.get("instance_type") or "t2.micro" +allowed_ssh_cidr = config.get("allowed_ssh_cidr") or "0.0.0.0/0" +ssh_public_key_path = config.get("ssh_public_key_path") or os.path.expanduser("~/.ssh/id_rsa.pub") + +with open(ssh_public_key_path) as f: + public_key_content = f.read() + +# Latest Ubuntu 22.04 LTS AMI +ami = aws.ec2.get_ami( + most_recent=True, + owners=["099720109477"], + filters=[ + aws.ec2.GetAmiFilterArgs(name="name", values=["ubuntu/images/hvm-ssd/ubuntu-jammy-22.04-amd64-server-*"]), + aws.ec2.GetAmiFilterArgs(name="virtualization-type", values=["hvm"]), + ], +) + +# SSH key pair +key_pair = aws.ec2.KeyPair( + "vm-key", + key_name=f"{project_name}-key", + public_key=public_key_content, +) + +# Security group: SSH (22), HTTP (80), app (5000) +sg = aws.ec2.SecurityGroup( + "vm-sg", + name=f"{project_name}-sg", + description="Allow SSH, HTTP, and app port 5000", + ingress=[ + aws.ec2.SecurityGroupIngressArgs(protocol="tcp", from_port=22, to_port=22, cidr_blocks=[allowed_ssh_cidr], description="SSH"), + aws.ec2.SecurityGroupIngressArgs(protocol="tcp", from_port=80, to_port=80, cidr_blocks=["0.0.0.0/0"], description="HTTP"), + aws.ec2.SecurityGroupIngressArgs(protocol="tcp", from_port=5000, to_port=5000, cidr_blocks=["0.0.0.0/0"], description="App"), + ], + egress=[aws.ec2.SecurityGroupEgressArgs(protocol="-1", from_port=0, to_port=0, cidr_blocks=["0.0.0.0/0"])], + tags={"Name": f"{project_name}-sg"}, +) + +# EC2 instance (free tier: t2.micro) +instance = aws.ec2.Instance( + "vm", + ami=ami.id, + instance_type=instance_type, + key_name=key_pair.key_name, + vpc_security_group_ids=[sg.id], + associate_public_ip_address=True, + root_block_device=aws.ec2.InstanceRootBlockDeviceArgs(volume_size=8, volume_type="gp2"), + user_data="""#!/bin/bash +apt-get update -y +apt-get install -y python3 +""", + tags={"Name": f"{project_name}-vm"}, +) + +pulumi.export("public_ip", instance.public_ip) +pulumi.export("instance_id", instance.id) +pulumi.export("ssh_command", instance.public_ip.apply(lambda ip: f"ssh ubuntu@{ip}")) diff --git a/pulumi/requirements.txt b/pulumi/requirements.txt new file mode 100644 index 0000000000..2b3f653eaf --- /dev/null +++ b/pulumi/requirements.txt @@ -0,0 +1,2 @@ +pulumi>=3.0.0 +pulumi-aws>=6.0.0 diff --git a/terraform/.gitignore b/terraform/.gitignore new file mode 100644 index 0000000000..7672ebf3af --- /dev/null +++ b/terraform/.gitignore @@ -0,0 +1,23 @@ +# Terraform +*.tfstate +*.tfstate.* +.terraform/ +.terraform.lock.hcl +terraform.tfvars +*.tfvars +!terraform.tfvars.example + +# Credentials +*.pem +*.key +*.json +credentials +override.tf +override.tf.json +*_override.tf +*_override.tf.json + +# Crash and debug +crash.log +crash.*.log +*.log diff --git a/terraform/.tflint.hcl b/terraform/.tflint.hcl new file mode 100644 index 0000000000..c1f91e49ed --- /dev/null +++ b/terraform/.tflint.hcl @@ -0,0 +1,7 @@ +plugin "terraform" { + enabled = true +} + +plugin "aws" { + enabled = true +} diff --git a/terraform/README.md b/terraform/README.md new file mode 100644 index 0000000000..ef779d408c --- /dev/null +++ b/terraform/README.md @@ -0,0 +1,41 @@ +# Terraform — Lab 4 IaC + +Provisions a single EC2 VM on AWS (free tier) with SSH, HTTP, and port 5000 open. + +## Prerequisites + +- [Terraform](https://developer.hashicorp.com/terraform/downloads) >= 1.9 +- AWS CLI configured, or env vars: `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY` +- SSH key pair (default: `~/.ssh/id_rsa.pub`) + +## Usage + +```bash +cd terraform + +# Copy example tfvars (optional) +cp terraform.tfvars.example terraform.tfvars +# Edit terraform.tfvars with your values (do not commit!) + +terraform init +terraform fmt +terraform validate +terraform plan +terraform apply +``` + +## Outputs + +- `public_ip` — VM public IP +- `ssh_command` — Example SSH command (user: `ubuntu`) + +## Cleanup + +```bash +terraform destroy +``` + +## Security + +- Restrict `allowed_ssh_cidr` to your IP (e.g. `"YOUR_IP/32"`). +- Never commit `terraform.tfvars` or `.tfstate`. diff --git a/terraform/docs/LAB04.md b/terraform/docs/LAB04.md new file mode 100644 index 0000000000..3c7be8a82b --- /dev/null +++ b/terraform/docs/LAB04.md @@ -0,0 +1,83 @@ +# LAB04 — Infrastructure as Code (Terraform & Pulumi) + +## 1. Cloud Provider & Infrastructure + +- **Cloud provider:** AWS + **Rationale:** Widely used, strong Terraform/Pulumi support, free tier (t2.micro, 750 hrs/month for 12 months). +- **Instance type/size:** `t2.micro` (1 vCPU, 1 GiB RAM) — AWS free tier. +- **Region/zone:** `us-east-1` (default; change via `aws_region` / `aws:region`). +- **Estimated cost:** $0 within free tier (ensure no other paid resources). +- **Resources created:** + - EC2 instance (Ubuntu 22.04 LTS) + - Security group (SSH 22, HTTP 80, app 5000) + - Key pair (from your SSH public key) + - Public IP (assigned by default to the instance) + +--- + +## 2. Terraform Implementation + +- **Terraform version:** 1.9+ +- **Project structure:** + - `main.tf` — provider, data source (AMI), key pair, security group, EC2 instance + - `variables.tf` — region, project name, instance type, SSH key path, allowed CIDR, tags + - `outputs.tf` — public IP, instance ID, SSH command + - `terraform.tfvars.example` — example variable values (copy to `terraform.tfvars`, gitignored) +- **Decisions:** Use default VPC; Ubuntu 22.04 AMI via data source; single security group for all rules. +- **Challenges:** (Document any you hit: e.g. AMI ownership, key path, region.) +- **Terminal output:** (Add your own sanitized output.) + - `terraform init` + - `terraform plan` (no secrets) + - `terraform apply` + - SSH connection (e.g. `ssh ubuntu@`) + +--- + +## 3. Pulumi Implementation + +- **Pulumi version:** 3.x | **Language:** Python +- **Differences from Terraform:** Same resources expressed in Python (imperative style); config via `pulumi config`; outputs via `pulumi.export()`. +- **Advantages:** Full Python (loops, conditionals, reuse); IDE support; encrypted secrets in Pulumi Cloud. +- **Challenges:** (Document any: e.g. SSH key path at startup, config vs Terraform variables.) +- **Terminal output:** (Add your own.) + - `pulumi preview` + - `pulumi up` + - SSH connection to Pulumi-created VM + +--- + +## 4. Terraform vs Pulumi Comparison + +| Aspect | Terraform | Pulumi | +|---------------|-----------------------------------|----------------------------------| +| **Ease of learning** | HCL is small and focused; good for simple infra. | Easier if you already know Python/TS. | +| **Readability** | Declarative; structure is clear. | Code can be more compact; logic is explicit. | +| **Debugging** | Plan/output and provider docs. | Stack traces and IDE help. | +| **Documentation** | Large community and registry. | Good docs; smaller ecosystem. | +| **Use case** | Standard choice for multi-cloud IaC, teams. | Strong when you want code reuse and tests. | + +**When to use Terraform:** Multi-cloud, team standardization, lots of examples and modules. +**When to use Pulumi:** Prefer coding in Python/TS, need loops/functions or testing in the same language. + +--- + +## 5. Lab 5 Preparation & Cleanup + +**VM for Lab 5:** +- [ ] Keeping VM for Lab 5? (Yes / No) +- [ ] If yes: Which one? (Terraform / Pulumi) +- [ ] If no: Plan for Lab 5? (Local VM / Recreate cloud VM later) + +**Cleanup status:** +- If keeping one VM: Note which tool created it and that the other stack has been destroyed. +- If destroying all: Run `terraform destroy` and `pulumi destroy`; add short terminal output (no secrets). +- Optional: Screenshot of cloud console showing no (or only intended) resources. + +--- + +## 6. Bonus: IaC CI/CD + +- **Workflow:** `.github/workflows/terraform-ci.yml` runs on changes to `terraform/**`. +- **Steps:** `terraform fmt -check`, `terraform init -backend=false`, `terraform validate`, `tflint --init` then `tflint`. +- **Path filters:** Only when `terraform/**` or the workflow file changes. +- A dummy SSH public key is created in CI so `file()` in Terraform does not fail during validate. diff --git a/terraform/main.tf b/terraform/main.tf new file mode 100644 index 0000000000..e9439e134f --- /dev/null +++ b/terraform/main.tf @@ -0,0 +1,105 @@ +terraform { + required_version = ">= 1.9.0" + + required_providers { + aws = { + source = "hashicorp/aws" + version = "~> 5.0" + } + } +} + +provider "aws" { + region = var.aws_region + + default_tags { + tags = var.tags + } +} + +# Latest Ubuntu 22.04 LTS AMI +data "aws_ami" "ubuntu" { + most_recent = true + owners = ["099720109477"] # Canonical + + filter { + name = "name" + values = ["ubuntu/images/hvm-ssd/ubuntu-jammy-22.04-amd64-server-*"] + } + + filter { + name = "virtualization-type" + values = ["hvm"] + } +} + +# SSH key pair (uses existing public key) +resource "aws_key_pair" "vm" { + key_name = "${var.project_name}-key" + public_key = file(pathexpand(var.ssh_public_key_path)) +} + +# Security group: SSH (22), HTTP (80), app (5000) +resource "aws_security_group" "vm" { + name = "${var.project_name}-sg" + description = "Allow SSH, HTTP, and app port 5000" + + ingress { + description = "SSH" + from_port = 22 + to_port = 22 + protocol = "tcp" + cidr_blocks = [var.allowed_ssh_cidr] + } + + ingress { + description = "HTTP" + from_port = 80 + to_port = 80 + protocol = "tcp" + cidr_blocks = ["0.0.0.0/0"] + } + + ingress { + description = "App port 5000" + from_port = 5000 + to_port = 5000 + protocol = "tcp" + cidr_blocks = ["0.0.0.0/0"] + } + + egress { + from_port = 0 + to_port = 0 + protocol = "-1" + cidr_blocks = ["0.0.0.0/0"] + } + + tags = { + Name = "${var.project_name}-sg" + } +} + +# EC2 instance (free tier: t2.micro) +resource "aws_instance" "vm" { + ami = data.aws_ami.ubuntu.id + instance_type = var.instance_type + key_name = aws_key_pair.vm.key_name + vpc_security_group_ids = [aws_security_group.vm.id] + associate_public_ip_address = true + + root_block_device { + volume_size = 8 + volume_type = "gp2" + } + + user_data = <<-EOT + #!/bin/bash + apt-get update -y + apt-get install -y python3 + EOT + + tags = { + Name = "${var.project_name}-vm" + } +} diff --git a/terraform/outputs.tf b/terraform/outputs.tf new file mode 100644 index 0000000000..cf104d32ed --- /dev/null +++ b/terraform/outputs.tf @@ -0,0 +1,14 @@ +output "public_ip" { + description = "Public IP address of the VM" + value = aws_instance.vm.public_ip +} + +output "instance_id" { + description = "EC2 instance ID" + value = aws_instance.vm.id +} + +output "ssh_command" { + description = "SSH command to connect to the VM" + value = "ssh -i ~/.ssh/id_rsa ubuntu@${aws_instance.vm.public_ip}" +} diff --git a/terraform/terraform.tfvars.example b/terraform/terraform.tfvars.example new file mode 100644 index 0000000000..3ba69f4466 --- /dev/null +++ b/terraform/terraform.tfvars.example @@ -0,0 +1,6 @@ +# Copy to terraform.tfvars and fill in (terraform.tfvars is gitignored) +# aws_region = "us-east-1" +# project_name = "devops-lab04" +# instance_type = "t2.micro" +# ssh_public_key_path = "~/.ssh/id_rsa.pub" +# allowed_ssh_cidr = "YOUR_IP/32" # e.g. "203.0.113.42/32" diff --git a/terraform/variables.tf b/terraform/variables.tf new file mode 100644 index 0000000000..8639c79c04 --- /dev/null +++ b/terraform/variables.tf @@ -0,0 +1,38 @@ +variable "aws_region" { + description = "AWS region for resources" + type = string + default = "us-east-1" +} + +variable "project_name" { + description = "Project name used for resource naming and tags" + type = string + default = "devops-lab04" +} + +variable "instance_type" { + description = "EC2 instance type (use t2.micro for free tier)" + type = string + default = "t2.micro" +} + +variable "ssh_public_key_path" { + description = "Path to your SSH public key for VM access" + type = string + default = "~/.ssh/id_rsa.pub" +} + +variable "allowed_ssh_cidr" { + description = "CIDR block allowed for SSH (restrict to your IP for security)" + type = string + default = "0.0.0.0/0" # Replace with your IP/32 in production +} + +variable "tags" { + description = "Common tags for all resources" + type = map(string) + default = { + Project = "DevOps-Core-Course" + Lab = "lab04" + } +}