diff --git a/.github/workflows/ansible-deploy.yml b/.github/workflows/ansible-deploy.yml new file mode 100644 index 0000000000..c2123d0349 --- /dev/null +++ b/.github/workflows/ansible-deploy.yml @@ -0,0 +1,124 @@ +name: Ansible Deployment - Python App + +on: + push: + branches: [ main, master, lab06 ] + paths: + - 'ansible/vars/app_python.yml' + - 'ansible/playbooks/deploy_python.yml' + - 'ansible/playbooks/deploy.yml' + - 'ansible/roles/web_app/**' + - 'ansible/roles/common/**' + - 'ansible/roles/docker/**' + - '.github/workflows/ansible-deploy.yml' + pull_request: + branches: [ main, master, lab06 ] + paths: + - 'ansible/vars/app_python.yml' + - 'ansible/playbooks/deploy_python.yml' + - 'ansible/roles/web_app/**' + workflow_dispatch: + +jobs: + lint: + name: Ansible Lint - Python App + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.12' + + - name: Install dependencies + run: | + pip install ansible ansible-lint + + - name: Run ansible-lint + run: | + cd ansible + ansible-lint playbooks/deploy_python.yml playbooks/deploy.yml + + deploy: + name: Deploy Python Application + needs: lint + runs-on: ubuntu-latest + if: github.event_name == 'push' || github.event_name == 'workflow_dispatch' + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.12' + + - name: Install Ansible and dependencies + run: | + pip install ansible + ansible-galaxy collection install community.docker + + - name: Setup SSH + run: | + if [ -z "${{ secrets.SSH_PRIVATE_KEY }}" ]; then + echo "Error: SSH_PRIVATE_KEY secret is not set" + echo "Please configure the SSH_PRIVATE_KEY secret in repository settings" + exit 1 + fi + if [ -z "${{ secrets.VM_HOST }}" ]; then + echo "Error: VM_HOST secret is not set" + echo "Please configure the VM_HOST secret in repository settings" + exit 1 + fi + + mkdir -p ~/.ssh + echo "${{ secrets.SSH_PRIVATE_KEY }}" > ~/.ssh/yandex_cloud_key + chmod 600 ~/.ssh/yandex_cloud_key + ssh-keyscan -H ${{ secrets.VM_HOST }} >> ~/.ssh/known_hosts + + - name: Create Vault password file + env: + ANSIBLE_VAULT_PASSWORD: ${{ secrets.ANSIBLE_VAULT_PASSWORD }} + run: | + if [ -z "$ANSIBLE_VAULT_PASSWORD" ]; then + echo "Error: ANSIBLE_VAULT_PASSWORD secret is not set" + echo "Please configure the ANSIBLE_VAULT_PASSWORD secret in repository settings" + exit 1 + fi + echo "$ANSIBLE_VAULT_PASSWORD" > /tmp/vault_pass + chmod 600 /tmp/vault_pass + + - name: Deploy Python App with Ansible + run: | + cd ansible + ansible-playbook playbooks/deploy_python.yml \ + --vault-password-file /tmp/vault_pass \ + --tags "app_deploy" \ + -e @group_vars/all.yml + + - name: Cleanup sensitive files + if: always() + run: | + rm -f /tmp/vault_pass + rm -f ~/.ssh/yandex_cloud_key + + - name: Verify Python App Deployment + run: | + echo "Waiting for application to start..." + sleep 10 + + echo "Testing main endpoint..." + if ! curl -f http://${{ secrets.VM_HOST }}:8000; then + echo "Error: Python app is not accessible at http://${{ secrets.VM_HOST }}:8000" + exit 1 + fi + + echo "Testing health endpoint..." + if ! curl -f http://${{ secrets.VM_HOST }}:8000/health; then + echo "Error: Python app health check failed at http://${{ secrets.VM_HOST }}:8000/health" + exit 1 + fi + + echo "Deployment verification successful!" diff --git a/.github/workflows/go-ci.yml b/.github/workflows/go-ci.yml new file mode 100644 index 0000000000..3fdc914fb8 --- /dev/null +++ b/.github/workflows/go-ci.yml @@ -0,0 +1,99 @@ +name: Go ci + +on: + push: + branches: ["lab03", "master"] + paths: + - "app_go/**" + - ".github/workflows/go-ci.yml" + pull_request: + branches: ["lab03", "master"] + paths: + - "app_go/**" + - ".github/workflows/go-ci.yml" + +jobs: + test: + runs-on: ubuntu-latest + defaults: + run: + working-directory: app_go + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-go@v5 + with: + go-version: "1.22" + cache: true + + - name: Go fmt check + run: | + test -z "$(gofmt -l .)" + + - name: Run tests with coverage + working-directory: app_go + run: go test -coverprofile=coverage.out ./... + + - name: Upload coverage artifact + uses: actions/upload-artifact@v4 + with: + name: go-coverage + path: app_go/coverage.out + + - name: Upload coverage to Codecov + uses: codecov/codecov-action@v5 + with: + files: app_go/coverage.out + flags: go + env: + CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} + + lint: + runs-on: ubuntu-latest + defaults: + run: + working-directory: app_go + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-go@v5 + with: + go-version: "1.22" + cache: true + + - name: golangci-lint + uses: golangci/golangci-lint-action@v6 + with: + version: v1.61.0 + working-directory: app_go + + docker: + needs: [test, lint] + runs-on: ubuntu-latest + if: github.event_name == 'push' && (github.ref == 'refs/heads/lab03' || github.ref == 'refs/heads/master') + steps: + - uses: actions/checkout@v4 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Login to Docker Hub + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_TOKEN }} + + - name: Generate version + run: echo "VERSION=$(date -u +%Y.%m.%d)" >> $GITHUB_ENV + + - name: Build and push + uses: docker/build-push-action@v6 + with: + context: ./app_go + push: true + tags: | + ${{ secrets.DOCKER_USERNAME }}/go_app:${{ env.VERSION }} + ${{ secrets.DOCKER_USERNAME }}/go_app:${{ github.sha }} + ${{ secrets.DOCKER_USERNAME }}/go_app:latest + cache-from: type=gha + cache-to: type=gha,mode=max diff --git a/.github/workflows/python-ci.yml b/.github/workflows/python-ci.yml new file mode 100644 index 0000000000..f0ed5ca75c --- /dev/null +++ b/.github/workflows/python-ci.yml @@ -0,0 +1,99 @@ +name: Python ci + + +on: + push: + branches: [ "lab03", "master" ] + paths: + - 'app_python/**' + - '.github/workflows/python-ci.yml' + pull_request: + branches: [ "lab03", "master" ] + paths: + - 'app_python/**' + - '.github/workflows/python-ci.yml' + +jobs: + test: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v4 + with: + python-version: 3.12 + cache: "pip" + cache-dependency-path: app_python/requirements.txt + + - name: Install dependencies + run: pip install -r app_python/requirements.txt + + - name: Run linter + run: flake8 + + - name: Run tests + working-directory: app_python + run: pytest --cov=. --cov-report=xml:coverage.xml --cov-report=term + + - name: Upload coverage to Codecov + uses: codecov/codecov-action@v5 + with: + files: app_python/coverage.xml + fail_ci_if_error: true + flags: python + env: + CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} + + security: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + cache: "pip" + cache-dependency-path: app_python/requirements.txt + + - name: Install dependencies + run: pip install -r app_python/requirements.txt + + - name: Setup Snyk CLI + uses: snyk/actions/setup@master + + - name: Run Snyk (dependencies) + working-directory: app_python + env: + SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }} + run: snyk test --severity-threshold=high + + docker: + needs: [ test, security ] + runs-on: ubuntu-latest + if: github.event_name == 'push' && (github.ref == 'refs/heads/lab03' || github.ref == 'refs/heads/master') + steps: + - uses: actions/checkout@v4 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Login to Docker Hub + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_TOKEN }} + + - name: Generate version + run: echo "VERSION=$(date -u +%Y.%m.%d)" >> $GITHUB_ENV + + - name: Build and push + uses: docker/build-push-action@v6 + with: + context: ./app_python + push: true + tags: | + ${{ secrets.DOCKER_USERNAME }}/python_app:${{ env.VERSION }} + ${{ secrets.DOCKER_USERNAME }}/python_app:${{ github.sha }} + ${{ secrets.DOCKER_USERNAME }}/python_app:latest + cache-from: type=gha + cache-to: type=gha,mode=max + diff --git a/.github/workflows/terraform-ci.yml b/.github/workflows/terraform-ci.yml new file mode 100644 index 0000000000..6931a90b72 --- /dev/null +++ b/.github/workflows/terraform-ci.yml @@ -0,0 +1,94 @@ +name: Terraform CI + +on: + pull_request: + paths: + - 'terraform/**' + - '.github/workflows/terraform-ci.yml' + push: + branches: + - lab04 + paths: + - 'terraform/**' + - '.github/workflows/terraform-ci.yml' + +jobs: + terraform-validate: + name: Terraform Validation + runs-on: ubuntu-latest + + defaults: + run: + working-directory: terraform + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Setup Terraform + uses: hashicorp/setup-terraform@v3 + with: + terraform_version: 1.9.0 + + - name: Terraform Format Check + id: fmt + run: terraform fmt -check -recursive + continue-on-error: true + + - name: Terraform Init + id: init + run: terraform init -backend=false + + - name: Terraform Validate + id: validate + run: | + terraform validate -no-color | tee validate.txt + + - name: Setup TFLint + uses: terraform-linters/setup-tflint@v4 + with: + tflint_version: latest + + - name: Initialize TFLint + run: tflint --init + + - name: Run TFLint + id: tflint + run: tflint --format compact + continue-on-error: true + + - name: Comment PR with Results + if: github.event_name == 'pull_request' + uses: actions/github-script@v7 + with: + script: | + const fs = require('fs'); + const validateOut = fs.existsSync('terraform/validate.txt') + ? fs.readFileSync('terraform/validate.txt', 'utf8') + : 'No validation output'; + + const output = `#### Terraform Format and Style πŸ–Œ\`${{ steps.fmt.outcome }}\` + #### Terraform Initialization βš™οΈ\`${{ steps.init.outcome }}\` + #### Terraform Validation πŸ€–\`${{ steps.validate.outcome }}\` + #### TFLint πŸ”\`${{ steps.tflint.outcome }}\` + +
Show Validation Output + + \`\`\` + ${validateOut} + \`\`\` + +
+ + *Pusher: @${{ github.actor }}, Action: \`${{ github.event_name }}\`, Workflow: \`${{ github.workflow }}\`*`; + + github.rest.issues.createComment({ + issue_number: context.issue.number, + owner: context.repo.owner, + repo: context.repo.repo, + body: output + }) + + - name: Fail if validation failed + if: steps.fmt.outcome == 'failure' || steps.validate.outcome == 'failure' + run: exit 1 \ No newline at end of file diff --git a/.gitignore b/.gitignore index 30d74d2584..e8f819db52 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,75 @@ -test \ No newline at end of file +# General +test +.DS_Store +*.log + +# Terraform +*.terraform.lock.hcl +*.tfstate +*.tfstate.* +.terraform/ +terraform.tfvars +*.tfvars +crash.log +crash.*.log +override.tf +override.tf.json +*_override.tf +*_override.tf.json + +# Pulumi +pulumi/venv/ +pulumi/__pycache__/ +Pulumi.*.yaml +.pulumi/ + +# Cloud credentials +*.pem +*.key +*.json +credentials +.aws/ +.azure/ +.gcp/ + +# IDE +.vscode/ +.idea/ +*.swp +*.swo +*~ + +# Python +__pycache__/ +*.py[cod] +*$py.class +*.so +.Python +env/ +venv/ +ENV/ +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +*.egg-info/ +.installed.cfg +*.egg + +# Node +node_modules/ +npm-debug.log + +# Ansible +*.retry +.vault_pass +ansible/inventory/*.pyc +yarn-error.log \ No newline at end of file diff --git a/ansible/README.md b/ansible/README.md new file mode 100644 index 0000000000..ae0f380d3c --- /dev/null +++ b/ansible/README.md @@ -0,0 +1,3 @@ +# Ansible Automation for DevOps Course + +[![Ansible Deployment - Python App](https://github.com/newspec/DevOps-Core-Course/actions/workflows/ansible-deploy.yml/badge.svg)](https://github.com/newspec/DevOps-Core-Course/actions/workflows/ansible-deploy.yml) diff --git a/ansible/ansible.cfg b/ansible/ansible.cfg new file mode 100644 index 0000000000..ee2655531e --- /dev/null +++ b/ansible/ansible.cfg @@ -0,0 +1,11 @@ +[defaults] +inventory = inventory/hosts.ini +roles_path = roles +host_key_checking = False +remote_user = ubuntu +retry_files_enabled = False + +[privilege_escalation] +become = True +become_method = sudo +become_user = root \ No newline at end of file diff --git a/ansible/docs/LAB05.md b/ansible/docs/LAB05.md new file mode 100644 index 0000000000..3b77f8b248 --- /dev/null +++ b/ansible/docs/LAB05.md @@ -0,0 +1,633 @@ +# Lab 5 - Ansible Fundamentals Documentation + +## 1. Architecture Overview + +### Ansible version used +``` +ansible [core 2.20.2] + config file = /Users/newspec/Desktop/DevOps/DevOps-Core-Course/ansible/ansible.cfg + configured module search path = ['/Users/newspec/.ansible/plugins/modules', '/usr/share/ansible/plugins/modules'] + ansible python module location = /opt/homebrew/Cellar/ansible/13.3.0/libexec/lib/python3.14/site-packages/ansible + ansible collection location = /Users/newspec/.ansible/collections:/usr/share/ansible/collections + executable location = /opt/homebrew/bin/ansible + python version = 3.14.3 (main, Feb 3 2026, 15:32:20) [Clang 17.0.0 (clang-1700.6.3.2)] (/opt/homebrew/Cellar/ansible/13.3.0/libexec/bin/python) + jinja version = 3.1.6 + pyyaml version = 6.0.3 (with libyaml v0.2.5) + ``` + +### Target VM OS and version +- **Operating System**: Ubuntu 24.04 LTS Vanilla + +### Role structure diagram or explanation + +This project uses a **role-based architecture** for maximum reusability and maintainability: + +``` +ansible/ +β”œβ”€β”€ roles/ +β”‚ β”œβ”€β”€ common/ # System provisioning (packages, timezone) +β”‚ β”œβ”€β”€ docker/ # Docker installation and configuration +β”‚ └── app_deploy/ # Application deployment with Docker +β”œβ”€β”€ playbooks/ +β”‚ β”œβ”€β”€ site.yml # Complete infrastructure + deployment +β”‚ β”œβ”€β”€ provision.yml # System provisioning only +β”‚ └── deploy.yml # Application deployment only +β”œβ”€β”€ inventory/ +β”‚ └── hosts.ini # Static inventory +└── group_vars/ + └── all.yml # Encrypted variables (Ansible Vault) +``` + +### Why Roles Instead of Monolithic Playbooks? + +**Roles provide:** + +1. **Reusability**: Same role can be used across multiple projects +2. **Modularity**: Each role has a single, well-defined purpose +3. **Maintainability**: Changes are isolated to specific roles +4. **Testability**: Roles can be tested independently +5. **Sharing**: Roles can be shared via Ansible Galaxy +6. **Organization**: Clear structure makes code easy to navigate +7. **Scalability**: Easy to add new roles without affecting existing ones + +**Example**: The `docker` role can be reused in any project that needs Docker, without modification. + +--- + +## 2. Roles Documentation + +### 2.1 Common Role + +**Purpose**: Performs basic system provisioning tasks that every server needs. + +**Key Variables** (from `defaults/main.yml`): +```yaml +common_packages: + - python3-pip + - curl + - git + - vim + - htop + - wget + - unzip + - software-properties-common + - apt-transport-https + - ca-certificates + - gnupg + - lsb-release + +timezone: "UTC" +``` + +**Tasks**: +1. Update apt cache (with 3600s cache validity) +2. Install common packages +3. Set system timezone + +**Handlers**: None (no services to restart) + +**Dependencies**: None + +**Tags**: `common`, `packages`, `timezone` + +--- + +### 2.2 Docker Role + +**Purpose**: Installs and configures Docker CE on Ubuntu systems. + +**Key Variables** (from `defaults/main.yml`): +```yaml +docker_user: "ubuntu" +docker_version: "latest" +``` + +**Tasks**: +1. Install Docker prerequisites +2. Create directory for Docker GPG key +3. Add Docker GPG key +4. Detect Ubuntu release codename +5. Add Docker repository +6. Install Docker packages (docker-ce, docker-ce-cli, containerd.io) +7. Ensure Docker service is running and enabled +8. Add user to docker group +9. Install python3-docker (for Ansible docker modules) +10. Verify Docker installation + +**Handlers**: +- `restart docker`: Restarts Docker service when repository or packages change + +**Dependencies**: None (but typically runs after `common` role) + +**Tags**: `docker`, `packages`, `service`, `user`, `verify` + +**Idempotency Features**: +- Uses `apt_key` and `apt_repository` modules (stateful) +- Service module ensures desired state +- User module only adds to group if not already present + +--- + +### 2.3 App Deploy Role + +**Purpose**: Deploys containerized Python application using Docker. + +**Key Variables** (from `defaults/main.yml`): +```yaml +app_port: 8000 +app_restart_policy: "unless-stopped" +app_environment_vars: {} +``` + +**Variables from Vault** (group_vars/all.yml): +```yaml +dockerhub_username: +dockerhub_password: +app_name: devops-app +docker_image: "{{ dockerhub_username }}/{{ app_name }}" +docker_image_tag: latest +app_container_name: "{{ app_name }}" +``` + +**Tasks**: +1. Log in to Docker Hub (credentials from Vault, `no_log: true`) +2. Pull Docker image +3. Stop existing container (if running) +4. Remove old container (if exists) +5. Run new container with proper configuration +6. Wait for application port to be available +7. Verify health endpoint +8. Display health check result + +**Handlers**: +- `restart application`: Restarts the application container + +**Dependencies**: Requires `docker` role to be run first + +**Tags**: `app`, `deploy`, `verify` + +**Security Features**: +- Uses `no_log: true` for Docker login (prevents credential exposure) +- Credentials stored in encrypted Ansible Vault +- Uses Docker Hub access tokens (not passwords) + +--- + +## 3. Idempotency Demonstration + +### Terminal output from FIRST provision.yml run + +```bash +PLAY [Provision web servers] ********************************************************************************************* + +TASK [Gathering Facts] *************************************************************************************************** +ok: [lab04-vm] + +TASK [common : Update apt cache] ***************************************************************************************** +changed: [lab04-vm] + +TASK [common : Install common packages] ********************************************************************************** +changed: [lab04-vm] + +TASK [common : Set timezone] ********************************************************************************************* +changed: [lab04-vm] + +TASK [docker : Install prerequisites for Docker] ************************************************************************* +ok: [lab04-vm] + +TASK [docker : Create directory for Docker GPG key] ********************************************************************** +ok: [lab04-vm] + +TASK [docker : Add Docker GPG key] *************************************************************************************** +changed: [lab04-vm] + +TASK [docker : Get Ubuntu release codename] ****************************************************************************** +ok: [lab04-vm] + +TASK [docker : Get system architecture] ********************************************************************************** +ok: [lab04-vm] + +TASK [docker : Add Docker repository] ************************************************************************************ +changed: [lab04-vm] + +TASK [docker : Install Docker packages] ********************************************************************************** +changed: [lab04-vm] + +TASK [docker : Ensure Docker service is running and enabled] ************************************************************* +ok: [lab04-vm] + +TASK [docker : Add user to docker group] ********************************************************************************* +changed: [lab04-vm] + +TASK [docker : Install python3-docker for Ansible docker modules] ******************************************************** +changed: [lab04-vm] + +TASK [docker : Verify Docker installation] ******************************************************************************* +ok: [lab04-vm] + +TASK [docker : Display Docker version] *********************************************************************************** +ok: [lab04-vm] => { + "msg": "Docker version installed: Docker version 29.2.1, build a5c7197" +} + +RUNNING HANDLER [docker : restart docker] ******************************************************************************** +changed: [lab04-vm] + +PLAY RECAP *************************************************************************************************************** +lab04-vm : ok=17 changed=9 unreachable=0 failed=0 skipped=0 rescued=0 ignored=0 +``` + +### Terminal output from SECOND provision.yml run + +```bash +PLAY [Provision web servers] ******************************************************************************************************************************************************************* + +TASK [Gathering Facts] ************************************************************************************************************************************************************************* +ok: [lab04-vm] + +TASK [common : Update apt cache] *************************************************************************************************************************************************************** +ok: [lab04-vm] + +TASK [common : Install common packages] ******************************************************************************************************************************************************** +ok: [lab04-vm] + +TASK [common : Set timezone] ******************************************************************************************************************************************************************* +ok: [lab04-vm] + +TASK [docker : Install prerequisites for Docker] *********************************************************************************************************************************************** +ok: [lab04-vm] + +TASK [docker : Create directory for Docker GPG key] ******************************************************************************************************************************************** +ok: [lab04-vm] + +TASK [docker : Add Docker GPG key] ************************************************************************************************************************************************************* +ok: [lab04-vm] + +TASK [docker : Get Ubuntu release codename] **************************************************************************************************************************************************** +ok: [lab04-vm] + +TASK [docker : Get system architecture] ******************************************************************************************************************************************************** +ok: [lab04-vm] + +TASK [docker : Add Docker repository] ********************************************************************************************************************************************************** +ok: [lab04-vm] + +TASK [docker : Install Docker packages] ******************************************************************************************************************************************************** +ok: [lab04-vm] + +TASK [docker : Ensure Docker service is running and enabled] *********************************************************************************************************************************** +ok: [lab04-vm] + +TASK [docker : Add user to docker group] ******************************************************************************************************************************************************* +ok: [lab04-vm] + +TASK [docker : Install python3-docker for Ansible docker modules] ****************************************************************************************************************************** +ok: [lab04-vm] + +TASK [docker : Verify Docker installation] ***************************************************************************************************************************************************** +ok: [lab04-vm] + +TASK [docker : Display Docker version] ********************************************************************************************************************************************************* +ok: [lab04-vm] => { + "msg": "Docker version installed: Docker version 29.2.1, build a5c7197" +} + +PLAY RECAP ************************************************************************************************************************************************************************************* +lab04-vm : ok=16 changed=0 unreachable=0 failed=0 skipped=0 rescued=0 ignored=0 +``` + +### Analysis: What Changed First Time? What Didn't Change Second Time? + +#### First Run Analysis (9 tasks changed) + +**Tasks That Changed (Yellow "changed" status):** + +1. **`common : Update apt cache`** - Changed because apt cache was outdated or didn't exist +2. **`common : Install common packages`** - Changed because packages were not installed yet +3. **`common : Set timezone`** - Changed because timezone was not set to UTC +4. **`docker : Add Docker GPG key`** - Changed because GPG key was not present in system +5. **`docker : Add Docker repository`** - Changed because Docker repository was not configured +6. **`docker : Install Docker packages`** - Changed because Docker was not installed +7. **`docker : Add user to docker group`** - Changed because user was not in docker group +8. **`docker : Install python3-docker`** - Changed because python3-docker package was not installed +9. **`HANDLER: restart docker`** - Executed because Docker repository and packages were added (notified by tasks) + +**Tasks That Didn't Change (Green "ok" status):** + +1. **`Gathering Facts`** - Always runs to collect system information (not a change) +2. **`docker : Install prerequisites for Docker`** - Prerequisites already present on Ubuntu 24.04 +3. **`docker : Create directory for Docker GPG key`** - Directory `/etc/apt/keyrings` already existed +4. **`docker : Get Ubuntu release codename`** - Read-only command (marked with `changed_when: false`) +5. **`docker : Get system architecture`** - Read-only command (marked with `changed_when: false`) +6. **`docker : Ensure Docker service is running and enabled`** - Service already running after installation +7. **`docker : Verify Docker installation`** - Read-only verification command +8. **`docker : Display Docker version`** - Debug task (always shows "ok") + +**Summary**: 9 out of 17 tasks made actual changes to the system. + +--- + +#### Second Run Analysis (0 tasks changed) + +**All Tasks Showed "ok" Status (Green):** + +Every task verified that the desired state already exists and made no changes: + +1. **`common : Update apt cache`** - Cache still valid (within 3600s window) +2. **`common : Install common packages`** - All packages already installed at correct versions +3. **`common : Set timezone`** - Timezone already set to UTC +4. **`docker : Install prerequisites for Docker`** - Prerequisites already present +5. **`docker : Create directory for Docker GPG key`** - Directory already exists +6. **`docker : Add Docker GPG key`** - GPG key already present in keyring +7. **`docker : Add Docker repository`** - Repository already configured in sources list +8. **`docker : Install Docker packages`** - Docker packages already installed at correct versions +9. **`docker : Ensure Docker service is running and enabled`** - Service already running and enabled +10. **`docker : Add user to docker group`** - User already member of docker group +11. **`docker : Install python3-docker`** - Package already installed + +**No Handlers Executed:** +- `restart docker` handler was NOT triggered because no tasks reported "changed" status +- Handlers only run when notified by tasks that make changes + +**Key Difference**: +- **First run**: `ok=17 changed=9` (9 changes + 1 handler) +- **Second run**: `ok=16 changed=0` (no changes, no handlers) + +--- + +### What Makes These Roles Idempotent? + +**1. Stateful Modules**: +- `apt`: Uses `state=present` (not `command: apt install`) +- `service`: Uses `state=started` (not `command: systemctl start`) +- `user`: Uses `groups` with `append=yes` +- `file`: Uses `state=directory` + +**2. Conditional Execution**: +- `cache_valid_time: 3600` prevents unnecessary apt updates +- `changed_when: false` for read-only commands +- `ignore_errors: yes` for cleanup tasks + +**3. Declarative Approach**: +- Describe desired state, not steps to achieve it +- Ansible determines what changes are needed +- Only executes necessary actions + +**4. Handler Pattern**: +- Handlers only run when notified +- Handlers only run once per play (even if notified multiple times) +- Handlers run at end of play (after all tasks) + +--- + +## 4. Ansible Vault Usage + +### How you store credentials securely + +**Encrypted File** (`group_vars/all.yml`): +``` +$ANSIBLE_VAULT;1.1;AES256 +64646331363832623830316636666263633265316239373332653234623633333434643135306638 +3639333565353461386639613533356433623135616634650a663166353437643861363063393166 +34393639333066326165333439653936613538333931663161303333663665656261663536346538 +3432306235663933310a313364323164633930306161396466333264626336393163623366666332 +62623333366637343830353337366364393437643264326466633839666466623633396130623435 +30626237653337323635653965393432316562333462356333333066636363373939326361316462 +37393662376630373266666135353432376536666436323866313530636234353733353232346539 +30663063316166376264626462356565393035373835636338343834343838623333376433633938 +32626663313033376465616462313934353431376532356334373762353264633338633438633366 +32383161663064636337396663636638343335363535343734333132373139343530366435323538 +64666239666133326632376531353663383534646538623135393962363564393632373265613565 +64313164636565633635376634396231646636353339626465303166386334663063633937363832 +35383236653731613538383236376233393063363736613237366564326266623135 +... +``` + +### Vault Password Management Strategy +Use password file (`.vault_pass`) for convenience, with strict `.gitignore` rules. + +### Example of encrypted file (show it's encrypted!) +**Encrypted File** (`group_vars/all.yml`): +``` +$ANSIBLE_VAULT;1.1;AES256 +64646331363832623830316636666263633265316239373332653234623633333434643135306638 +3639333565353461386639613533356433623135616634650a663166353437643861363063393166 +34393639333066326165333439653936613538333931663161303333663665656261663536346538 +3432306235663933310a313364323164633930306161396466333264626336393163623366666332 +62623333366637343830353337366364393437643264326466633839666466623633396130623435 +30626237653337323635653965393432316562333462356333333066636363373939326361316462 +37393662376630373266666135353432376536666436323866313530636234353733353232346539 +30663063316166376264626462356565393035373835636338343834343838623333376433633938 +32626663313033376465616462313934353431376532356334373762353264633338633438633366 +32383161663064636337396663636638343335363535343734333132373139343530366435323538 +64666239666133326632376531353663383534646538623135393962363564393632373265613565 +64313164636565633635376634396231646636353339626465303166386334663063633937363832 +35383236653731613538383236376233393063363736613237366564326266623135 +... +``` + +### Why Ansible Vault is Important + +**Security Risks Without Vault:** + +1. **Credential Exposure in Version Control** + - Plain text passwords visible in Git history + - Anyone with repository access can see credentials + - Credentials remain in history even after deletion + - Public repositories expose secrets to the world + +2. **Compliance Violations** + - Most security standards prohibit plain text credentials + - GDPR, PCI-DSS, SOC 2 require encrypted credential storage + - Audit failures if secrets found in version control + - Legal and financial consequences + +3. **Insider Threats** + - All team members can access all credentials + - No access control or audit trail + - Difficult to rotate credentials + - Former employees retain access through Git history + +4. **Accidental Exposure** + - Easy to accidentally commit secrets + - Secrets can leak through logs, screenshots, or error messages + - CI/CD pipelines may expose credentials + - Third-party integrations may access repository + +--- + +## 5. Deployment Verification + +### Terminal output from deploy.yml run + +```bash +PLAY [Deploy application] ********************************************************************************************************************************************************************** + +TASK [Gathering Facts] ************************************************************************************************************************************************************************* +ok: [lab04-vm] + +TASK [app_deploy : Log in to Docker Hub] ******************************************************************************************************************************************************* +ok: [lab04-vm] + +TASK [app_deploy : Pull Docker image] ********************************************************************************************************************************************************** +ok: [lab04-vm] + +TASK [app_deploy : Stop and remove existing container (if exists)] ***************************************************************************************************************************** +changed: [lab04-vm] + +TASK [app_deploy : Run new container] ********************************************************************************************************************************************************** +changed: [lab04-vm] + +TASK [app_deploy : Wait for application port to be available] ********************************************************************************************************************************** +ok: [lab04-vm] + +TASK [app_deploy : Verify health endpoint] ***************************************************************************************************************************************************** +ok: [lab04-vm] + +TASK [app_deploy : Display health check result] ************************************************************************************************************************************************ +ok: [lab04-vm] => { + "msg": "Application is healthy: {'status': 'healthy', 'timestamp': '2026-02-21T18:07:25.200649+00:00', 'uptime_seconds': 5}" +} + +RUNNING HANDLER [app_deploy : restart application] ********************************************************************************************************************************************* +changed: [lab04-vm] + +PLAY RECAP ************************************************************************************************************************************************************************************* +lab04-vm : ok=9 changed=3 unreachable=0 failed=0 skipped=0 rescued=0 ignored=0 +``` +--- + +### Container status: docker ps output + +```bash +lab04-vm | CHANGED | rc=0 >> +CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES +a08611d9d5bf newspec/python_app:latest "python app.py" About a minute ago Up 52 seconds 0.0.0.0:8000->8000/tcp python_app +``` + +--- + +### Health check verification: curl outputs + +```bash +lab04-vm | CHANGED | rc=0 >> +{"status":"healthy","timestamp":"2026-02-21T18:10:04.436185+00:00","uptime_seconds":156} +``` + +--- + +### Handler Execution + +**When Handlers Run**: +- Handlers are triggered by `notify` in tasks +- Handlers only run if task reports "changed" +- Handlers run at end of play (after all tasks) +- Each handler runs only once per play + +**Example**: If Docker repository is added (changed), the `restart docker` handler is notified and runs at the end. + +**In Our Deployment**: +- `restart application` handler **WAS triggered** and executed +- Handler was notified by the "Run new container" task (line 479) which reported "changed" +- The handler ran at the end of the play, after all tasks completed +- This demonstrates proper handler usage: the container was started, then the handler ensured it was properly restarted with all configurations applied + +**Why the handler triggered**: +The "Run new container" task includes `notify: restart application` in [`roles/app_deploy/tasks/main.yml`](../roles/app_deploy/tasks/main.yml:45). Since this task reported "changed" status (the container was newly created), the handler was notified and executed at the end of the play. + +--- + +## 6. Key Decisions + +### Why use roles instead of plain playbooks? + +Roles provide **modularity and reusability**. Instead of one large playbook with all tasks, we have: +- **Separate concerns**: Each role has one purpose +- **Reusable components**: Docker role can be used in any project +- **Easy maintenance**: Changes to Docker installation only affect docker role +- **Clear structure**: Standard directory layout makes code easy to understand +- **Independent testing**: Each role can be tested separately + +**Example**: If we need to add MongoDB to our infrastructure, we create a new `mongodb` role without touching existing roles. + +--- + +### How do roles improve reusability? + +1. **Parameterization**: Variables in `defaults/main.yml` make roles configurable +2. **No hardcoding**: Roles use variables, not hardcoded values +3. **Standard structure**: Anyone familiar with Ansible can understand our roles +4. **Ansible Galaxy**: Roles can be shared publicly or privately +5. **Version control**: Roles can be versioned independently + +**Example**: The `docker` role can be used to install Docker on any Ubuntu system, just by including it in a playbook. + +--- + +### What makes a task idempotent? + +A task is idempotent when: +1. **Uses stateful modules**: `apt`, `service`, `user`, `file` (not `command` or `shell`) +2. **Describes desired state**: "Package should be present" not "Install package" +3. **Checks before acting**: Module checks current state before making changes +4. **No side effects**: Running twice doesn't cause problems + +**Example**: +```yaml +# βœ… Idempotent +- name: Install nginx + apt: + name: nginx + state: present + +# ❌ Not idempotent +- name: Install nginx + command: apt install -y nginx +``` + +The first task checks if nginx is installed before installing. The second always runs `apt install`. + +--- + +### How do handlers improve efficiency? + +Handlers provide **smart service management**: + +1. **Deferred execution**: Handlers run at end of play, not immediately +2. **Run once**: Even if notified multiple times, handler runs only once +3. **Conditional execution**: Only run if notified (i.e., if something changed) +4. **Grouped restarts**: Multiple changes trigger one restart + +**Example**: If we update Docker repository AND install Docker packages, both tasks notify `restart docker`, but Docker only restarts once at the end. + +**Without handlers**: We'd restart Docker after each change (inefficient). + +**With handlers**: Docker restarts once after all changes (efficient). + +--- + +### Why is Ansible Vault necessary? + +Ansible Vault is **essential for security**: + +1. **Protects credentials**: Passwords, API keys, tokens encrypted +2. **Safe version control**: Encrypted files can be committed to Git +3. **Compliance**: Meets security requirements for credential storage +4. **Audit trail**: Changes to encrypted files tracked in Git +5. **Access control**: Only those with vault password can decrypt + +**Without Vault**: Credentials in plain text, visible in Git history, easily exposed. + +**With Vault**: Credentials encrypted, safe to commit, only accessible with password. + +**Real-world scenario**: If our Git repository is compromised, attackers cannot access our Docker Hub credentials because they're encrypted. + +--- + +## 7. Challenges and Solutions + +### Challenge 1: Docker Repository Architecture Detection + +**Issue**: Docker repository URL needs correct architecture (amd64 vs arm64). + +**Solution**: Dynamic definition of architecture. diff --git a/ansible/docs/LAB06.md b/ansible/docs/LAB06.md new file mode 100644 index 0000000000..a47f2b355e --- /dev/null +++ b/ansible/docs/LAB06.md @@ -0,0 +1,3770 @@ +# Lab 6: Advanced Ansible & CI/CD +--- + +## Overview + +This lab enhanced the Ansible automation from Lab 5 with production-ready features including: +- **Blocks and Tags** for better task organization and selective execution +- **Docker Compose** for declarative container management +- **Role Dependencies** for automatic prerequisite handling +- **Wipe Logic** with double-gating (variable + tag) for safe cleanup +- **CI/CD Integration** with GitHub Actions for automated deployments + +**Technologies Used:** +- Ansible 2.16+ +- Docker Compose v2 +- GitHub Actions +- Jinja2 templating +- Ansible Vault for secrets management + +--- + +## Task 1: Blocks & Tags + +### 1.1 Common Role Refactoring + +**File:** [`roles/common/tasks/main.yml`](../roles/common/tasks/main.yml) + +**Implementation:** +- Grouped package installation tasks in a block with `packages` tag +- Added rescue block for apt cache update failures +- Implemented always block to log completion +- Applied `become: true` at block level for efficiency + +**Block Structure:** +```yaml +- name: Package installation tasks + block: + - name: Update apt cache + - name: Install common packages + rescue: + - name: Handle apt cache update failure + - name: Fix missing packages + - name: Retry package installation + always: + - name: Log package installation completion + become: true + tags: + - common + - packages +``` + +**Tag Strategy:** +- `common` - entire role (applied at role level) +- `packages` - all package installation tasks +- `config` - system configuration tasks + +### 1.2 Docker Role Refactoring + +**File:** [`roles/docker/tasks/main.yml`](../roles/docker/tasks/main.yml) + +**Implementation:** +- Grouped Docker installation tasks in block with `docker_install` tag +- Grouped Docker configuration tasks in block with `docker_config` tag +- Added rescue block to retry GPG key addition on network timeout +- Used always block to ensure Docker service is enabled + +**Block Structure:** +```yaml +- name: Docker installation tasks + block: + - name: Install prerequisites + - name: Add Docker GPG key + - name: Add Docker repository + - name: Install Docker packages + rescue: + - name: Handle Docker installation failure + - name: Wait before retry + - name: Retry Docker GPG key addition + - name: Retry Docker package installation + always: + - name: Ensure Docker service is enabled and started + - name: Log Docker installation completion + become: true + tags: + - docker + - docker_install +``` + +**Tag Strategy:** +- `docker` - entire role +- `docker_install` - installation only +- `docker_config` - configuration only + +### 1.3 Output showing selective execution with --tags + +```bash +# Test provision with only docker +ansible-playbook playbooks/provision.yml --tags "docker" +PLAY [Provision web servers] ************************************************************************************************************************************************ + +TASK [Gathering Facts] ****************************************************************************************************************************************************** +ok: [lab04-vm] + +TASK [docker : Install prerequisites for Docker] **************************************************************************************************************************** +ok: [lab04-vm] + +TASK [docker : Create directory for Docker GPG key] ************************************************************************************************************************* +ok: [lab04-vm] + +TASK [docker : Add Docker GPG key] ****************************************************************************************************************************************** +ok: [lab04-vm] + +TASK [docker : Get Ubuntu release codename] ********************************************************************************************************************************* +ok: [lab04-vm] + +TASK [docker : Get system architecture] ************************************************************************************************************************************* +ok: [lab04-vm] + +TASK [docker : Add Docker repository] *************************************************************************************************************************************** +ok: [lab04-vm] + +TASK [docker : Install Docker packages] ************************************************************************************************************************************* +ok: [lab04-vm] + +TASK [docker : Install python3-docker for Ansible docker modules] *********************************************************************************************************** +ok: [lab04-vm] + +TASK [docker : Ensure Docker service is enabled and started] **************************************************************************************************************** +ok: [lab04-vm] + +TASK [docker : Log Docker installation completion] ************************************************************************************************************************** +[WARNING]: Deprecation warnings can be disabled by setting `deprecation_warnings=False` in ansible.cfg. +[DEPRECATION WARNING]: INJECT_FACTS_AS_VARS default to `True` is deprecated, top-level facts will not be auto injected after the change. This feature will be removed from ansible-core version 2.24. +Origin: /Users/newspec/Desktop/DevOps/DevOps-Core-Course/ansible/roles/docker/tasks/main.yml:99:18 + +97 - name: Log Docker installation completion +98 copy: +99 content: "Docker installation completed at {{ ansible_date_time.iso8601 }}\n" + ^ column 18 + +Use `ansible_facts["fact_name"]` (no `ansible_` prefix) instead. + +changed: [lab04-vm] + +TASK [docker : Add user to docker group] ************************************************************************************************************************************ +ok: [lab04-vm] + +TASK [docker : Verify Docker installation] ********************************************************************************************************************************** +ok: [lab04-vm] + +TASK [docker : Display Docker version] ************************************************************************************************************************************** +ok: [lab04-vm] => { + "msg": "Docker version installed: Docker version 29.2.1, build a5c7197" +} + +TASK [docker : Log Docker configuration completion] ************************************************************************************************************************* +[DEPRECATION WARNING]: INJECT_FACTS_AS_VARS default to `True` is deprecated, top-level facts will not be auto injected after the change. This feature will be removed from ansible-core version 2.24. +Origin: /Users/newspec/Desktop/DevOps/DevOps-Core-Course/ansible/roles/docker/tasks/main.yml:129:18 + +127 - name: Log Docker configuration completion +128 copy: +129 content: "Docker configuration completed at {{ ansible_date_time.iso8601 }}\n" + ^ column 18 + +Use `ansible_facts["fact_name"]` (no `ansible_` prefix) instead. + +changed: [lab04-vm] + +PLAY RECAP ****************************************************************************************************************************************************************** +lab04-vm : ok=15 changed=2 unreachable=0 failed=0 skipped=0 rescued=0 ignored=0 +``` +```bash +# Skip common role +ansible-playbook playbooks/provision.yml --skip-tags "common" +PLAY [Provision web servers] ************************************************************************************************************************************************ + +TASK [Gathering Facts] ****************************************************************************************************************************************************** +ok: [lab04-vm] + +TASK [docker : Install prerequisites for Docker] **************************************************************************************************************************** +ok: [lab04-vm] + +TASK [docker : Create directory for Docker GPG key] ************************************************************************************************************************* +ok: [lab04-vm] + +TASK [docker : Add Docker GPG key] ****************************************************************************************************************************************** +ok: [lab04-vm] + +TASK [docker : Get Ubuntu release codename] ********************************************************************************************************************************* +ok: [lab04-vm] + +TASK [docker : Get system architecture] ************************************************************************************************************************************* +ok: [lab04-vm] + +TASK [docker : Add Docker repository] *************************************************************************************************************************************** +ok: [lab04-vm] + +TASK [docker : Install Docker packages] ************************************************************************************************************************************* +ok: [lab04-vm] + +TASK [docker : Install python3-docker for Ansible docker modules] *********************************************************************************************************** +ok: [lab04-vm] + +TASK [docker : Ensure Docker service is enabled and started] **************************************************************************************************************** +ok: [lab04-vm] + +TASK [docker : Log Docker installation completion] ************************************************************************************************************************** +[WARNING]: Deprecation warnings can be disabled by setting `deprecation_warnings=False` in ansible.cfg. +[DEPRECATION WARNING]: INJECT_FACTS_AS_VARS default to `True` is deprecated, top-level facts will not be auto injected after the change. This feature will be removed from ansible-core version 2.24. +Origin: /Users/newspec/Desktop/DevOps/DevOps-Core-Course/ansible/roles/docker/tasks/main.yml:99:18 + +97 - name: Log Docker installation completion +98 copy: +99 content: "Docker installation completed at {{ ansible_date_time.iso8601 }}\n" + ^ column 18 + +Use `ansible_facts["fact_name"]` (no `ansible_` prefix) instead. + +changed: [lab04-vm] + +TASK [docker : Add user to docker group] ************************************************************************************************************************************ +ok: [lab04-vm] + +TASK [docker : Verify Docker installation] ********************************************************************************************************************************** +ok: [lab04-vm] + +TASK [docker : Display Docker version] ************************************************************************************************************************************** +ok: [lab04-vm] => { + "msg": "Docker version installed: Docker version 29.2.1, build a5c7197" +} + +TASK [docker : Log Docker configuration completion] ************************************************************************************************************************* +[DEPRECATION WARNING]: INJECT_FACTS_AS_VARS default to `True` is deprecated, top-level facts will not be auto injected after the change. This feature will be removed from ansible-core version 2.24. +Origin: /Users/newspec/Desktop/DevOps/DevOps-Core-Course/ansible/roles/docker/tasks/main.yml:129:18 + +127 - name: Log Docker configuration completion +128 copy: +129 content: "Docker configuration completed at {{ ansible_date_time.iso8601 }}\n" + ^ column 18 + +Use `ansible_facts["fact_name"]` (no `ansible_` prefix) instead. + +changed: [lab04-vm] + +PLAY RECAP ****************************************************************************************************************************************************************** +lab04-vm : ok=15 changed=2 unreachable=0 failed=0 skipped=0 rescued=0 ignored=0 +``` +```bash +# Install packages only across all roles +ansible-playbook playbooks/provision.yml --tags "packages" +PLAY [Provision web servers] ************************************************************************************************************************************************ + +TASK [Gathering Facts] ****************************************************************************************************************************************************** +ok: [lab04-vm] + +TASK [common : Update apt cache] ******************************************************************************************************************************************** +ok: [lab04-vm] + +TASK [common : Install common packages] ************************************************************************************************************************************* +ok: [lab04-vm] + +TASK [common : Log package installation completion] ************************************************************************************************************************* +[WARNING]: Deprecation warnings can be disabled by setting `deprecation_warnings=False` in ansible.cfg. +[DEPRECATION WARNING]: INJECT_FACTS_AS_VARS default to `True` is deprecated, top-level facts will not be auto injected after the change. This feature will be removed from ansible-core version 2.24. +Origin: /Users/newspec/Desktop/DevOps/DevOps-Core-Course/ansible/roles/common/tasks/main.yml:36:18 + +34 - name: Log package installation completion +35 copy: +36 content: "Common packages installation completed at {{ ansible_date_time.iso8601 }}\n" + ^ column 18 + +Use `ansible_facts["fact_name"]` (no `ansible_` prefix) instead. + +changed: [lab04-vm] + +PLAY RECAP ****************************************************************************************************************************************************************** +lab04-vm : ok=4 changed=1 unreachable=0 failed=0 skipped=0 rescued=0 ignored=0 +``` +```bash +# Check mode to see what would run +ansible-playbook playbooks/provision.yml --tags "docker" --check +PLAY [Provision web servers] ************************************************************************************************************************************************ + +TASK [Gathering Facts] ****************************************************************************************************************************************************** +ok: [lab04-vm] + +TASK [docker : Install prerequisites for Docker] **************************************************************************************************************************** +ok: [lab04-vm] + +TASK [docker : Create directory for Docker GPG key] ************************************************************************************************************************* +ok: [lab04-vm] + +TASK [docker : Add Docker GPG key] ****************************************************************************************************************************************** +ok: [lab04-vm] + +TASK [docker : Get Ubuntu release codename] ********************************************************************************************************************************* +skipping: [lab04-vm] + +TASK [docker : Get system architecture] ************************************************************************************************************************************* +skipping: [lab04-vm] + +TASK [docker : Add Docker repository] *************************************************************************************************************************************** +changed: [lab04-vm] + +TASK [docker : Install Docker packages] ************************************************************************************************************************************* +ok: [lab04-vm] + +TASK [docker : Install python3-docker for Ansible docker modules] *********************************************************************************************************** +ok: [lab04-vm] + +TASK [docker : Ensure Docker service is enabled and started] **************************************************************************************************************** +ok: [lab04-vm] + +TASK [docker : Log Docker installation completion] ************************************************************************************************************************** +[WARNING]: Deprecation warnings can be disabled by setting `deprecation_warnings=False` in ansible.cfg. +[DEPRECATION WARNING]: INJECT_FACTS_AS_VARS default to `True` is deprecated, top-level facts will not be auto injected after the change. This feature will be removed from ansible-core version 2.24. +Origin: /Users/newspec/Desktop/DevOps/DevOps-Core-Course/ansible/roles/docker/tasks/main.yml:99:18 + +97 - name: Log Docker installation completion +98 copy: +99 content: "Docker installation completed at {{ ansible_date_time.iso8601 }}\n" + ^ column 18 + +Use `ansible_facts["fact_name"]` (no `ansible_` prefix) instead. + +changed: [lab04-vm] + +TASK [docker : Add user to docker group] ************************************************************************************************************************************ +ok: [lab04-vm] + +TASK [docker : Verify Docker installation] ********************************************************************************************************************************** +skipping: [lab04-vm] + +TASK [docker : Display Docker version] ************************************************************************************************************************************** +ok: [lab04-vm] => { + "msg": "Docker version installed: " +} + +TASK [docker : Log Docker configuration completion] ************************************************************************************************************************* +[DEPRECATION WARNING]: INJECT_FACTS_AS_VARS default to `True` is deprecated, top-level facts will not be auto injected after the change. This feature will be removed from ansible-core version 2.24. +Origin: /Users/newspec/Desktop/DevOps/DevOps-Core-Course/ansible/roles/docker/tasks/main.yml:129:18 + +127 - name: Log Docker configuration completion +128 copy: +129 content: "Docker configuration completed at {{ ansible_date_time.iso8601 }}\n" + ^ column 18 + +Use `ansible_facts["fact_name"]` (no `ansible_` prefix) instead. + +changed: [lab04-vm] + +RUNNING HANDLER [docker : restart docker] *********************************************************************************************************************************** +changed: [lab04-vm] + +PLAY RECAP ****************************************************************************************************************************************************************** +lab04-vm : ok=13 changed=4 unreachable=0 failed=0 skipped=3 rescued=0 ignored=0 +``` +```bash +# Run only docker installation tasks +ansible-playbook playbooks/provision.yml --tags "docker_install" +PLAY [Provision web servers] ************************************************************************************************************************************************ + +TASK [Gathering Facts] ****************************************************************************************************************************************************** +ok: [lab04-vm] + +TASK [docker : Install prerequisites for Docker] **************************************************************************************************************************** +ok: [lab04-vm] + +TASK [docker : Create directory for Docker GPG key] ************************************************************************************************************************* +ok: [lab04-vm] + +TASK [docker : Add Docker GPG key] ****************************************************************************************************************************************** +ok: [lab04-vm] + +TASK [docker : Get Ubuntu release codename] ********************************************************************************************************************************* +ok: [lab04-vm] + +TASK [docker : Get system architecture] ************************************************************************************************************************************* +ok: [lab04-vm] + +TASK [docker : Add Docker repository] *************************************************************************************************************************************** +ok: [lab04-vm] + +TASK [docker : Install Docker packages] ************************************************************************************************************************************* +ok: [lab04-vm] + +TASK [docker : Install python3-docker for Ansible docker modules] *********************************************************************************************************** +ok: [lab04-vm] + +TASK [docker : Ensure Docker service is enabled and started] **************************************************************************************************************** +ok: [lab04-vm] + +TASK [docker : Log Docker installation completion] ************************************************************************************************************************** +[WARNING]: Deprecation warnings can be disabled by setting `deprecation_warnings=False` in ansible.cfg. +[DEPRECATION WARNING]: INJECT_FACTS_AS_VARS default to `True` is deprecated, top-level facts will not be auto injected after the change. This feature will be removed from ansible-core version 2.24. +Origin: /Users/newspec/Desktop/DevOps/DevOps-Core-Course/ansible/roles/docker/tasks/main.yml:99:18 + +97 - name: Log Docker installation completion +98 copy: +99 content: "Docker installation completed at {{ ansible_date_time.iso8601 }}\n" + ^ column 18 + +Use `ansible_facts["fact_name"]` (no `ansible_` prefix) instead. + +changed: [lab04-vm] + +PLAY RECAP ****************************************************************************************************************************************************************** +lab04-vm : ok=11 changed=1 unreachable=0 failed=0 skipped=0 rescued=0 ignored=0 +``` + +### 1.4 Output showing error handling with rescue block triggered +```bash +ansible-playbook playbooks/test_rescue.yml + LAY [Test rescue block in common role] *********************************************************************************************************************** + +TASK [Gathering Facts] **************************************************************************************************************************************** +ok: [lab04-vm] + +TASK [Backup original sources.list] *************************************************************************************************************************** +changed: [lab04-vm] + +TASK [Add invalid repository to trigger error] **************************************************************************************************************** +changed: [lab04-vm] + +TASK [Try to update apt cache (will fail)] ******************************************************************************************************************** +[WARNING]: Failed to update cache after 1 retries due to , retrying +[WARNING]: Sleeping for 1 seconds, before attempting to refresh the cache again +[WARNING]: Failed to update cache after 2 retries due to , retrying +[WARNING]: Sleeping for 2 seconds, before attempting to refresh the cache again +[WARNING]: Failed to update cache after 3 retries due to , retrying +[WARNING]: Sleeping for 4 seconds, before attempting to refresh the cache again +[WARNING]: Failed to update cache after 4 retries due to , retrying +[WARNING]: Sleeping for 8 seconds, before attempting to refresh the cache again +[WARNING]: Failed to update cache after 5 retries due to , retrying +[WARNING]: Sleeping for 12 seconds, before attempting to refresh the cache again +[ERROR]: Task failed: Module failed: Failed to update apt cache after 5 retries: +Origin: /Users/newspec/Desktop/DevOps/DevOps-Core-Course/ansible/playbooks/test_rescue.yml:21:11 + +19 state: present +20 +21 - name: Try to update apt cache (will fail) + ^ column 11 + +fatal: [lab04-vm]: FAILED! => {"changed": false, "msg": "Failed to update apt cache after 5 retries: "} + +TASK [Rescue block activated!] ******************************************************************************************************************************** +ok: [lab04-vm] => { + "msg": "RESCUE BLOCK TRIGGERED! Fixing apt sources..." +} + +TASK [Restore original sources.list] ************************************************************************************************************************** +changed: [lab04-vm] + +TASK [Run apt-get update --fix-missing] *********************************************************************************************************************** +changed: [lab04-vm] + +TASK [Retry apt update] *************************************************************************************************************************************** +changed: [lab04-vm] + +TASK [Cleanup backup file] ************************************************************************************************************************************ +changed: [lab04-vm] + +TASK [Always block executed] ********************************************************************************************************************************** +ok: [lab04-vm] => { + "msg": "Always block runs regardless of success/failure" +} + +PLAY RECAP **************************************************************************************************************************************************** +lab04-vm : ok=9 changed=6 unreachable=0 failed=0 skipped=0 rescued=1 ignored=0 +``` + +### 1.5 List of all available tags (--list-tags output) +```bash +ansible-playbook playbooks/provision.yml --list-tags + +playbook: playbooks/provision.yml + + play #1 (webservers): Provision web servers TAGS: [] + TASK TAGS: [common, config, docker, docker_config, docker_install, packages, users] +``` + +### 1.6 Research Questions Answered + +**Q: What happens if rescue block also fails?** +A: If the rescue block fails, the entire block fails and Ansible will stop execution (unless `ignore_errors: yes` is set). The always block will still execute before failure. This is why rescue blocks should be simple and reliable. + +**Q: Can you have nested blocks?** +A: Yes, blocks can be nested. However, it's generally not recommended as it makes playbooks harder to read. Better to use separate blocks or include_tasks for complex logic. + +**Q: How do tags inherit to tasks within blocks?** +A: Tags applied to a block are automatically inherited by all tasks within that block. This is more efficient than tagging each task individually. Child tasks can have additional tags beyond the block's tags. + +--- + +## Task 2: Docker Compose Migration (3 pts) + +### 2.1 Template Structure + +**File:** [`roles/web_app/templates/docker-compose.yml.j2`](../roles/web_app/templates/docker-compose.yml.j2) + +**Template Features:** +- Jinja2 templating for dynamic values +- Service name, image, ports, environment variables +- Restart policy: `unless-stopped` +- Custom bridge network for isolation +- Support for variable substitution + +**Key Variables:** +- `web_app_name` - service/container name (default: devops-app) +- `web_app_docker_image` - Docker Hub image +- `web_app_docker_tag` - image version (default: latest) +- `web_app_port` - host port (default: 8000) +- `web_app_internal_port` - container port (default: 8000) +- `web_app_environment_vars` - dictionary of environment variables + +### 2.2 Role Dependencies + +**File:** [`roles/web_app/meta/main.yml`](../roles/web_app/meta/main.yml) + +**Implementation:** +```yaml +dependencies: + - role: docker +``` + +**Purpose:** +- Ensures Docker is installed before deploying web app +- Automatic execution order without explicit playbook configuration +- Prevents deployment failures due to missing Docker + +**Test Result:** +Running only `web_app` role automatically executes `docker` role first. + +### 2.3 Before/After Comparison + +#### Before: Docker Run Approach (Lab 5) + +**File:** `roles/app_deploy/tasks/main.yml` + +```yaml +- name: Log in to Docker Hub + community.docker.docker_login: + username: "{{ dockerhub_username }}" + password: "{{ dockerhub_password }}" + no_log: true + +- name: Pull Docker image + community.docker.docker_image: + name: "{{ docker_image }}" + tag: "{{ docker_image_tag }}" + source: pull + +- name: Stop and remove existing container + community.docker.docker_container: + name: "{{ app_container_name }}" + state: absent + ignore_errors: yes + +- name: Run new container + community.docker.docker_container: + name: "{{ app_container_name }}" + image: "{{ docker_image }}:{{ docker_image_tag }}" + state: started + restart_policy: "{{ app_restart_policy }}" + ports: + - "{{ app_port }}:{{ app_port }}" + env: "{{ app_environment_vars | default({}) }}" +``` + +**Limitations:** +- Imperative approach (manual steps) +- No declarative configuration file +- Difficult to manage multiple containers +- No built-in networking between services +- Environment variables scattered in playbook +- Hard to version control container configuration +- Manual port mapping management + +#### After: Docker Compose Approach (Lab 6) + +**File:** `roles/web_app/tasks/main.yml` + +```yaml +- name: Create application directory + file: + path: "{{ web_app_compose_project_dir }}" + state: directory + mode: '0755' + +- name: Template docker-compose file + template: + src: docker-compose.yml.j2 + dest: "{{ web_app_compose_project_dir }}/docker-compose.yml" + mode: '0644' + +- name: Log in to Docker Hub + community.docker.docker_login: + username: "{{ dockerhub_username }}" + password: "{{ dockerhub_password }}" + no_log: true + +- name: Deploy with Docker Compose + community.docker.docker_compose_v2: + project_src: "{{ web_app_compose_project_dir }}" + state: present + pull: yes + recreate: smart +``` + +**File:** `roles/web_app/templates/docker-compose.yml.j2` + +```yaml +services: + {{ web_app_name }}: + image: {{ web_app_docker_image }}:{{ web_app_docker_tag }} + container_name: {{ web_app_name }} + ports: + - "{{ web_app_port }}:{{ web_app_internal_port }}" +{% if web_app_environment_vars is defined and web_app_environment_vars %} + environment: +{% for key, value in web_app_environment_vars.items() %} + {{ key }}: "{{ value }}" +{% endfor %} +{% endif %} + restart: {{ web_app_restart_policy }} + networks: + - app_network + +networks: + app_network: + driver: bridge +``` + +**Advantages:** +- Declarative configuration (infrastructure as code) +- Version-controlled compose file +- Easy multi-container management +- Built-in networking and service discovery +- Environment variables in one place +- Idempotent with `recreate: smart` +- Easier to understand and maintain +- Industry standard for container orchestration +- Supports volumes, networks, dependencies +- Better for production deployments + +#### Key Improvements + +| Aspect | Before (docker run) | After (Docker Compose) | +|--------|---------------------|------------------------| +| **Configuration** | Scattered in tasks | Centralized in compose file | +| **Idempotency** | Manual state management | Built-in with `recreate: smart` | +| **Networking** | Manual port mapping | Automatic network creation | +| **Environment** | Inline in playbook | Template with Jinja2 | +| **Multi-container** | Complex, error-prone | Simple, declarative | +| **Versioning** | Hard to track | Easy with compose file | +| **Rollback** | Manual container recreation | Simple file revert | +| **Production Ready** | Basic | Industry standard | + +#### Deployment Comparison + +**Before (5 tasks):** +1. Login to Docker Hub +2. Pull image +3. Stop old container +4. Remove old container +5. Run new container + +**After (4 tasks):** +1. Create directory +2. Template compose file +3. Login to Docker Hub +4. Deploy with compose (handles pull, stop, remove, start automatically) + +**Result:** Simpler, more maintainable, production-ready deployment! + +### 2.4 Research Questions Answered + +**Q: What's the difference between `restart: always` and `restart: unless-stopped`?** +A: +- `always`: Container restarts even if manually stopped, including after system reboot +- `unless-stopped`: Container restarts automatically EXCEPT when manually stopped. After reboot, it won't start if it was manually stopped before +- `unless-stopped` is better for production as it respects manual intervention + +**Q: How do Docker Compose networks differ from Docker bridge networks?** +A: +- Docker Compose creates isolated networks per project by default +- Compose networks have automatic DNS resolution between services +- Bridge networks are more manual and require explicit linking +- Compose networks are automatically cleaned up when project is removed + +**Q: Can you reference Ansible Vault variables in the template?** +A: Yes! Vault variables are decrypted before template rendering, so they can be used like any other variable in Jinja2 templates. Example: `{{ vault_secret_key }}` + +## 2.5 Output showing Docker Compose deployment success +```bash +PLAY [Deploy application] *************************************************************************************************************************************************** + +TASK [Gathering Facts] ****************************************************************************************************************************************************** +ok: [lab04-vm] + +TASK [docker : Install prerequisites for Docker] **************************************************************************************************************************** +ok: [lab04-vm] + +TASK [docker : Create directory for Docker GPG key] ************************************************************************************************************************* +ok: [lab04-vm] + +TASK [docker : Add Docker GPG key] ****************************************************************************************************************************************** +ok: [lab04-vm] + +TASK [docker : Get Ubuntu release codename] ********************************************************************************************************************************* +ok: [lab04-vm] + +TASK [docker : Get system architecture] ************************************************************************************************************************************* +ok: [lab04-vm] + +TASK [docker : Add Docker repository] *************************************************************************************************************************************** +ok: [lab04-vm] + +TASK [docker : Install Docker packages] ************************************************************************************************************************************* +ok: [lab04-vm] + +TASK [docker : Install python3-docker for Ansible docker modules] *********************************************************************************************************** +ok: [lab04-vm] + +TASK [docker : Ensure Docker service is enabled and started] **************************************************************************************************************** +ok: [lab04-vm] + +TASK [docker : Log Docker installation completion] ************************************************************************************************************************** +[WARNING]: Deprecation warnings can be disabled by setting `deprecation_warnings=False` in ansible.cfg. +[DEPRECATION WARNING]: INJECT_FACTS_AS_VARS default to `True` is deprecated, top-level facts will not be auto injected after the change. This feature will be removed from ansible-core version 2.24. +Origin: /Users/newspec/Desktop/DevOps/DevOps-Core-Course/ansible/roles/docker/tasks/main.yml:99:18 + +97 - name: Log Docker installation completion +98 copy: +99 content: "Docker installation completed at {{ ansible_date_time.iso8601 }}\n" + ^ column 18 + +Use `ansible_facts["fact_name"]` (no `ansible_` prefix) instead. + +changed: [lab04-vm] + +TASK [docker : Add user to docker group] ************************************************************************************************************************************ +ok: [lab04-vm] + +TASK [docker : Verify Docker installation] ********************************************************************************************************************************** +ok: [lab04-vm] + +TASK [docker : Display Docker version] ************************************************************************************************************************************** +ok: [lab04-vm] => { + "msg": "Docker version installed: Docker version 29.2.1, build a5c7197" +} + +TASK [docker : Log Docker configuration completion] ************************************************************************************************************************* +[DEPRECATION WARNING]: INJECT_FACTS_AS_VARS default to `True` is deprecated, top-level facts will not be auto injected after the change. This feature will be removed from ansible-core version 2.24. +Origin: /Users/newspec/Desktop/DevOps/DevOps-Core-Course/ansible/roles/docker/tasks/main.yml:129:18 + +127 - name: Log Docker configuration completion +128 copy: +129 content: "Docker configuration completed at {{ ansible_date_time.iso8601 }}\n" + ^ column 18 + +Use `ansible_facts["fact_name"]` (no `ansible_` prefix) instead. + +changed: [lab04-vm] + +TASK [web_app : Include wipe tasks] ***************************************************************************************************************************************** +included: /Users/newspec/Desktop/DevOps/DevOps-Core-Course/ansible/roles/web_app/tasks/wipe.yml for lab04-vm + +TASK [web_app : Stop and remove containers with Docker Compose] ************************************************************************************************************* +skipping: [lab04-vm] + +TASK [web_app : Remove docker-compose file] ********************************************************************************************************************************* +skipping: [lab04-vm] + +TASK [web_app : Remove application directory] ******************************************************************************************************************************* +skipping: [lab04-vm] + +TASK [web_app : Log wipe completion] **************************************************************************************************************************************** +skipping: [lab04-vm] + +TASK [web_app : Check if application is already running] ******************************************************************************************************************** +ok: [lab04-vm] + +TASK [web_app : Display current container status] *************************************************************************************************************************** +ok: [lab04-vm] => { + "msg": "Container devops-python is running" +} + +TASK [web_app : Create application directory] ******************************************************************************************************************************* +ok: [lab04-vm] + +TASK [web_app : Template docker-compose file] ******************************************************************************************************************************* +ok: [lab04-vm] + +TASK [web_app : Log in to Docker Hub] *************************************************************************************************************************************** +ok: [lab04-vm] + +TASK [web_app : Deploy with Docker Compose] ********************************************************************************************************************************* +ok: [lab04-vm] + +TASK [web_app : Display deployment result] ********************************************************************************************************************************** +ok: [lab04-vm] => { + "msg": "Deployment unchanged - containers are up to date" +} + +TASK [web_app : Wait for application port to be available (on target VM)] *************************************************************************************************** +ok: [lab04-vm] + +TASK [web_app : Verify health endpoint (from target VM)] ******************************************************************************************************************** +ok: [lab04-vm] + +TASK [web_app : Display health check result] ******************************************************************************************************************************** +ok: [lab04-vm] => { + "msg": "Application is healthy: {'status': 'healthy', 'timestamp': '2026-03-01T17:34:13.948071+00:00', 'uptime_seconds': 147}" +} + +PLAY RECAP ****************************************************************************************************************************************************************** +lab04-vm : ok=26 changed=2 unreachable=0 failed=0 skipped=4 rescued=0 ignored=0 +``` + +## 2.6 Idempotency proof (second run shows "ok" not "changed") +First run: +``` bash +PLAY [Deploy application] *************************************************************************************************************************************************** + +TASK [Gathering Facts] ****************************************************************************************************************************************************** +ok: [lab04-vm] + +TASK [docker : Install prerequisites for Docker] **************************************************************************************************************************** +ok: [lab04-vm] + +TASK [docker : Create directory for Docker GPG key] ************************************************************************************************************************* +ok: [lab04-vm] + +TASK [docker : Add Docker GPG key] ****************************************************************************************************************************************** +ok: [lab04-vm] + +TASK [docker : Get Ubuntu release codename] ********************************************************************************************************************************* +ok: [lab04-vm] + +TASK [docker : Get system architecture] ************************************************************************************************************************************* +ok: [lab04-vm] + +TASK [docker : Add Docker repository] *************************************************************************************************************************************** +changed: [lab04-vm] + +TASK [docker : Install Docker packages] ************************************************************************************************************************************* +changed: [lab04-vm] + +TASK [docker : Install python3-docker for Ansible docker modules] *********************************************************************************************************** +ok: [lab04-vm] + +TASK [docker : Ensure Docker service is enabled and started] **************************************************************************************************************** +ok: [lab04-vm] + +TASK [docker : Log Docker installation completion] ************************************************************************************************************************** +[WARNING]: Deprecation warnings can be disabled by setting `deprecation_warnings=False` in ansible.cfg. +[DEPRECATION WARNING]: INJECT_FACTS_AS_VARS default to `True` is deprecated, top-level facts will not be auto injected after the change. This feature will be removed from ansible-core version 2.24. +Origin: /Users/newspec/Desktop/DevOps/DevOps-Core-Course/ansible/roles/docker/tasks/main.yml:99:18 + +97 - name: Log Docker installation completion +98 copy: +99 content: "Docker installation completed at {{ ansible_date_time.iso8601 }}\n" + ^ column 18 + +Use `ansible_facts["fact_name"]` (no `ansible_` prefix) instead. + +changed: [lab04-vm] + +TASK [docker : Add user to docker group] ************************************************************************************************************************************ +changed: [lab04-vm] + +TASK [docker : Verify Docker installation] ********************************************************************************************************************************** +ok: [lab04-vm] + +TASK [docker : Display Docker version] ************************************************************************************************************************************** +ok: [lab04-vm] => { + "msg": "Docker version installed: Docker version 29.2.1, build a5c7197" +} + +TASK [docker : Log Docker configuration completion] ************************************************************************************************************************* +[DEPRECATION WARNING]: INJECT_FACTS_AS_VARS default to `True` is deprecated, top-level facts will not be auto injected after the change. This feature will be removed from ansible-core version 2.24. +Origin: /Users/newspec/Desktop/DevOps/DevOps-Core-Course/ansible/roles/docker/tasks/main.yml:129:18 + +127 - name: Log Docker configuration completion +128 copy: +129 content: "Docker configuration completed at {{ ansible_date_time.iso8601 }}\n" + ^ column 18 + +Use `ansible_facts["fact_name"]` (no `ansible_` prefix) instead. + +changed: [lab04-vm] + +TASK [web_app : Include wipe tasks] ***************************************************************************************************************************************** +included: /Users/newspec/Desktop/DevOps/DevOps-Core-Course/ansible/roles/web_app/tasks/wipe.yml for lab04-vm + +TASK [web_app : Check if docker-compose file exists] ************************************************************************************************************************ +skipping: [lab04-vm] + +TASK [web_app : Stop and remove containers with Docker Compose] ************************************************************************************************************* +skipping: [lab04-vm] + +TASK [web_app : Stop container manually if compose file doesn't exist] ****************************************************************************************************** +skipping: [lab04-vm] + +TASK [web_app : Remove docker-compose file] ********************************************************************************************************************************* +skipping: [lab04-vm] + +TASK [web_app : Remove application directory] ******************************************************************************************************************************* +skipping: [lab04-vm] + +TASK [web_app : Log wipe completion] **************************************************************************************************************************************** +skipping: [lab04-vm] + +TASK [web_app : Check if application is already running] ******************************************************************************************************************** +ok: [lab04-vm] + +TASK [web_app : Display current container status] *************************************************************************************************************************** +ok: [lab04-vm] => { + "msg": "Container devops-python is not running" +} + +TASK [web_app : Create application directory] ******************************************************************************************************************************* +changed: [lab04-vm] + +TASK [web_app : Template docker-compose file] ******************************************************************************************************************************* +changed: [lab04-vm] + +TASK [web_app : Log in to Docker Hub] *************************************************************************************************************************************** +ok: [lab04-vm] + +TASK [web_app : Deploy with Docker Compose] ********************************************************************************************************************************* +changed: [lab04-vm] + +TASK [web_app : Display deployment result] ********************************************************************************************************************************** +ok: [lab04-vm] => { + "msg": "Deployment changed - containers are updated" +} + +TASK [web_app : Wait for application port to be available (on target VM)] *************************************************************************************************** +ok: [lab04-vm] + +TASK [web_app : Verify health endpoint (from target VM)] ******************************************************************************************************************** +ok: [lab04-vm] + +TASK [web_app : Display health check result] ******************************************************************************************************************************** +ok: [lab04-vm] => { + "msg": "Application is healthy: {'status': 'healthy', 'timestamp': '2026-03-01T17:46:21.068443+00:00', 'uptime_seconds': 6}" +} + +RUNNING HANDLER [docker : restart docker] *********************************************************************************************************************************** +changed: [lab04-vm] + +PLAY RECAP ****************************************************************************************************************************************************************** +lab04-vm : ok=27 changed=9 unreachable=0 failed=0 skipped=6 rescued=0 ignored=0 +``` +Second run: +```bash +PLAY [Deploy application] *************************************************************************************************************************************************** + +TASK [Gathering Facts] ****************************************************************************************************************************************************** +ok: [lab04-vm] + +TASK [docker : Install prerequisites for Docker] **************************************************************************************************************************** +ok: [lab04-vm] + +TASK [docker : Create directory for Docker GPG key] ************************************************************************************************************************* +ok: [lab04-vm] + +TASK [docker : Add Docker GPG key] ****************************************************************************************************************************************** +ok: [lab04-vm] + +TASK [docker : Get Ubuntu release codename] ********************************************************************************************************************************* +ok: [lab04-vm] + +TASK [docker : Get system architecture] ************************************************************************************************************************************* +ok: [lab04-vm] + +TASK [docker : Add Docker repository] *************************************************************************************************************************************** +ok: [lab04-vm] + +TASK [docker : Install Docker packages] ************************************************************************************************************************************* +ok: [lab04-vm] + +TASK [docker : Install python3-docker for Ansible docker modules] *********************************************************************************************************** +ok: [lab04-vm] + +TASK [docker : Ensure Docker service is enabled and started] **************************************************************************************************************** +ok: [lab04-vm] + +TASK [docker : Log Docker installation completion] ************************************************************************************************************************** +[WARNING]: Deprecation warnings can be disabled by setting `deprecation_warnings=False` in ansible.cfg. +[DEPRECATION WARNING]: INJECT_FACTS_AS_VARS default to `True` is deprecated, top-level facts will not be auto injected after the change. This feature will be removed from ansible-core version 2.24. +Origin: /Users/newspec/Desktop/DevOps/DevOps-Core-Course/ansible/roles/docker/tasks/main.yml:99:18 + +97 - name: Log Docker installation completion +98 copy: +99 content: "Docker installation completed at {{ ansible_date_time.iso8601 }}\n" + ^ column 18 + +Use `ansible_facts["fact_name"]` (no `ansible_` prefix) instead. + +changed: [lab04-vm] + +TASK [docker : Add user to docker group] ************************************************************************************************************************************ +ok: [lab04-vm] + +TASK [docker : Verify Docker installation] ********************************************************************************************************************************** +ok: [lab04-vm] + +TASK [docker : Display Docker version] ************************************************************************************************************************************** +ok: [lab04-vm] => { + "msg": "Docker version installed: Docker version 29.2.1, build a5c7197" +} + +TASK [docker : Log Docker configuration completion] ************************************************************************************************************************* +[DEPRECATION WARNING]: INJECT_FACTS_AS_VARS default to `True` is deprecated, top-level facts will not be auto injected after the change. This feature will be removed from ansible-core version 2.24. +Origin: /Users/newspec/Desktop/DevOps/DevOps-Core-Course/ansible/roles/docker/tasks/main.yml:129:18 + +127 - name: Log Docker configuration completion +128 copy: +129 content: "Docker configuration completed at {{ ansible_date_time.iso8601 }}\n" + ^ column 18 + +Use `ansible_facts["fact_name"]` (no `ansible_` prefix) instead. + +changed: [lab04-vm] + +TASK [web_app : Include wipe tasks] ***************************************************************************************************************************************** +included: /Users/newspec/Desktop/DevOps/DevOps-Core-Course/ansible/roles/web_app/tasks/wipe.yml for lab04-vm + +TASK [web_app : Check if docker-compose file exists] ************************************************************************************************************************ +skipping: [lab04-vm] + +TASK [web_app : Stop and remove containers with Docker Compose] ************************************************************************************************************* +skipping: [lab04-vm] + +TASK [web_app : Stop container manually if compose file doesn't exist] ****************************************************************************************************** +skipping: [lab04-vm] + +TASK [web_app : Remove docker-compose file] ********************************************************************************************************************************* +skipping: [lab04-vm] + +TASK [web_app : Remove application directory] ******************************************************************************************************************************* +skipping: [lab04-vm] + +TASK [web_app : Log wipe completion] **************************************************************************************************************************************** +skipping: [lab04-vm] + +TASK [web_app : Check if application is already running] ******************************************************************************************************************** +ok: [lab04-vm] + +TASK [web_app : Display current container status] *************************************************************************************************************************** +ok: [lab04-vm] => { + "msg": "Container devops-python is running" +} + +TASK [web_app : Create application directory] ******************************************************************************************************************************* +ok: [lab04-vm] + +TASK [web_app : Template docker-compose file] ******************************************************************************************************************************* +ok: [lab04-vm] + +TASK [web_app : Log in to Docker Hub] *************************************************************************************************************************************** +ok: [lab04-vm] + +TASK [web_app : Deploy with Docker Compose] ********************************************************************************************************************************* +ok: [lab04-vm] + +TASK [web_app : Display deployment result] ********************************************************************************************************************************** +ok: [lab04-vm] => { + "msg": "Deployment unchanged - containers are up to date" +} + +TASK [web_app : Wait for application port to be available (on target VM)] *************************************************************************************************** +ok: [lab04-vm] + +TASK [web_app : Verify health endpoint (from target VM)] ******************************************************************************************************************** +ok: [lab04-vm] + +TASK [web_app : Display health check result] ******************************************************************************************************************************** +ok: [lab04-vm] => { + "msg": "Application is healthy: {'status': 'healthy', 'timestamp': '2026-03-01T17:48:42.025713+00:00', 'uptime_seconds': 135}" +} + +PLAY RECAP ****************************************************************************************************************************************************************** +lab04-vm : ok=26 changed=2 unreachable=0 failed=0 skipped=6 rescued=0 ignored=0 +``` + +## 2.7 Application running and accessible +```bash +ubuntu@lab04-vm:~$ docker ps +CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES +1253f65bdeb0 newspec/python_app:latest "python app.py" 4 minutes ago Up 4 minutes 0.0.0.0:8000->8000/tcp, [::]:8000->8000/tcp devops-python +ubuntu@lab04-vm:~$ docker compose -f /opt/devops-python/docker-compose.yml ps +NAME IMAGE COMMAND SERVICE CREATED STATUS PORTS +devops-python newspec/python_app:latest "python app.py" devops-python 5 minutes ago Up 4 minutes 0.0.0.0:8000->8000/tcp, [::]:8000->8000/tcp +ubuntu@lab04-vm:~$ curl http://localhost:8000 +{"service":{"name":"devops-info-service","version":"1.0.0","description":"DevOps course info service","framework":"FastAPI"},"system":{"hostname":"1253f65bdeb0","platform":"Linux","platform_version":"#100-Ubuntu SMP PREEMPT_DYNAMIC Tue Jan 13 16:40:06 UTC 2026","architecture":"x86_64","cpu_count":2,"python_version":"3.12.12"},"runtime":{"uptime_seconds":290,"uptime_human":"0 hours, 4 minutes","current_time":"2026-03-01T17:51:16.359911+00:00","timezone":"UTC"},"request":{"client_ip":"172.19.0.1","user_agent":"curl/8.5.0","method":"GET","path":"/"},"endpoints":[{"path":"/","method":"GET","description":"Service information"},{"path":"/health","method":"GET","description":"Health check"} +``` + +## 2.8 Contents of templated docker-compose.yml + +**File:** `/opt/devops-python/docker-compose.yml` (generated from template) + +```yaml +services: + devops-python: + image: newspec/python_app:latest + container_name: devops-python + ports: + - "8000:8000" + environment: + APP_NAME: "DevOps Python Service" + APP_VERSION: "1.0.0" + restart: unless-stopped + networks: + - app_network + +networks: + app_network: + driver: bridge +``` + +**Key Features:** +- **Service name:** `devops-python` (from `{{ web_app_name }}` variable) +- **Image:** `newspec/python_app:latest` (from `{{ web_app_docker_image }}:{{ web_app_docker_tag }}`) +- **Container name:** `devops-python` (matches service name) +- **Port mapping:** `8000:8000` (host:container) +- **Environment variables:** Templated from `web_app_environment_vars` dictionary +- **Restart policy:** `unless-stopped` (respects manual stops) +- **Network:** Custom bridge network `app_network` for isolation + +**Template Variables Used:** +- `web_app_name` = `devops-python` +- `web_app_docker_image` = `newspec/python_app` +- `web_app_docker_tag` = `latest` +- `web_app_port` = `8000` +- `web_app_internal_port` = `8000` +- `web_app_restart_policy` = `unless-stopped` +- `web_app_environment_vars.APP_NAME` = `"DevOps Python Service"` +- `web_app_environment_vars.APP_VERSION` = `"1.0.0"` + +--- + +## Task 3: Wipe Logic + +### 3.1 Implementation Details + +**Purpose:** Safe, controlled removal of deployed applications for: +- Clean reinstallation (wipe old β†’ deploy new) +- Testing from fresh state +- Rolling back to clean slate +- Resource cleanup before upgrades + +**Safety Mechanism:** Double-gating +1. Variable control: `web_app_wipe: true` +2. Tag control: `--tags web_app_wipe` + +**Complete Wipe Logic Implementation:** + +**File:** [`roles/web_app/tasks/wipe.yml`](../roles/web_app/tasks/wipe.yml) + +```yaml +--- +# Wipe logic for web application +- name: Wipe web application + block: + - name: Check if docker-compose file exists + stat: + path: "{{ web_app_compose_project_dir }}/docker-compose.yml" + register: compose_file + + - name: Stop and remove containers with Docker Compose + community.docker.docker_compose_v2: + project_src: "{{ web_app_compose_project_dir }}" + state: absent + when: compose_file.stat.exists + ignore_errors: yes + + - name: Stop container manually if compose file doesn't exist + community.docker.docker_container: + name: "{{ web_app_name }}" + state: absent + when: not compose_file.stat.exists + ignore_errors: yes + + - name: Remove docker-compose file + file: + path: "{{ web_app_compose_project_dir }}/docker-compose.yml" + state: absent + ignore_errors: yes + + - name: Remove application directory + file: + path: "{{ web_app_compose_project_dir }}" + state: absent + ignore_errors: yes + + - name: Log wipe completion + debug: + msg: "Application {{ web_app_name }} wiped successfully from {{ web_app_compose_project_dir }}" + + when: web_app_wipe | bool + become: true + tags: + - web_app_wipe +``` + +**Default Variable:** + +**File:** [`roles/web_app/defaults/main.yml`](../roles/web_app/defaults/main.yml) + +```yaml +# Wipe Logic Control +web_app_wipe: false # Default: do not wipe + +# Usage examples: +# Wipe only: ansible-playbook deploy.yml -e "web_app_wipe=true" --tags web_app_wipe +# Clean install: ansible-playbook deploy.yml -e "web_app_wipe=true" +``` + +**Key Implementation Features:** + +1. **Smart Container Removal:** + - Checks if docker-compose.yml exists before using compose module + - Falls back to docker_container module if compose file missing + - Handles both Docker Compose and standalone container scenarios + +2. **Error Handling:** + - All removal tasks have `ignore_errors: yes` + - Prevents failure if resources already removed + - Ensures wipe completes even if some steps fail + +3. **Complete Cleanup:** + - Stops and removes containers (via compose or direct) + - Removes docker-compose.yml file + - Removes entire application directory (/opt/devops-python) + - Logs completion for audit trail + +4. **Double-Gating Safety:** + - `when: web_app_wipe | bool` - Variable must be true + - `tags: web_app_wipe` - Tag must be specified + - **Both** conditions required for execution + +5. **Privilege Escalation:** + - `become: true` at block level + - Required for /opt directory operations + +### 3.2 Variable + Tag Approach + +**Double-Gating Safety Mechanism** + +The wipe logic uses a **two-layer safety mechanism** to prevent accidental data loss: + +#### Layer 1: Variable Control (`when` condition) + +```yaml +when: web_app_wipe | bool +``` + +**Purpose:** Prevents wipe tasks from running unless explicitly enabled + +**How it works:** +- Default value: `web_app_wipe: false` (in `defaults/main.yml`) +- Tasks only execute when variable is `true` +- Must be explicitly set via `-e` flag or vars file +- `| bool` filter ensures proper boolean evaluation + +**Example:** +```bash +# Variable is false (default) - wipe tasks SKIPPED +ansible-playbook deploy.yml --tags web_app_wipe +# Result: Wipe tasks skipped due to when condition + +# Variable is true - wipe tasks CAN run (if tag also specified) +ansible-playbook deploy.yml -e "web_app_wipe=true" --tags web_app_wipe +# Result: Wipe tasks execute +``` + +#### Layer 2: Tag Control + +```yaml +tags: + - web_app_wipe +``` + +**Purpose:** Requires explicit tag specification to include wipe tasks + +**How it works:** +- Wipe tasks have unique tag `web_app_wipe` +- Tasks only included when tag is specified with `--tags` +- Without tag, wipe tasks are not even considered for execution +- Provides command-line level safety + +**Example:** +```bash +# Tag not specified - wipe tasks NOT INCLUDED +ansible-playbook deploy.yml -e "web_app_wipe=true" +# Result: Wipe tasks run BEFORE deployment (clean install) + +# Tag specified - wipe tasks INCLUDED +ansible-playbook deploy.yml -e "web_app_wipe=true" --tags web_app_wipe +# Result: ONLY wipe tasks run, deployment skipped +``` + +#### Combined Effect: Double-Gating + +**Truth Table:** + +| Variable (`web_app_wipe`) | Tag (`--tags web_app_wipe`) | Result | +|---------------------------|----------------------------|--------| +| `false` (default) | Not specified | Wipe skipped, deployment runs | +| `false` (default) | Specified | Wipe skipped (when condition), deployment runs | +| `true` | Not specified | Wipe runs, then deployment runs (clean install) | +| `true` | Specified | Wipe runs, deployment skipped (wipe only) | + +**Use Cases:** + +1. **Normal Deployment** (no wipe): + ```bash + ansible-playbook deploy.yml + # Variable: false, Tag: not specified + # Result: Deployment only + ``` + +2. **Wipe Only** (remove app, no deployment): + ```bash + ansible-playbook deploy.yml -e "web_app_wipe=true" --tags web_app_wipe + # Variable: true, Tag: specified + # Result: Wipe only + ``` + +3. **Clean Reinstall** (wipe then deploy): + ```bash + ansible-playbook deploy.yml -e "web_app_wipe=true" + # Variable: true, Tag: not specified + # Result: Wipe β†’ Deploy + ``` + +4. **Safety Check** (tag without variable): + ```bash + ansible-playbook deploy.yml --tags web_app_wipe + # Variable: false, Tag: specified + # Result: Wipe skipped, deployment runs + ``` + +#### Why Both Layers? + +**Variable alone is not enough:** +- Could be accidentally set in vars file +- No command-line confirmation required +- Easy to forget it's enabled + +**Tag alone is not enough:** +- Could be run without realizing consequences +- No explicit "yes, I want to delete" confirmation +- Easier to mistype or autocomplete + +**Together they provide:** +- **Explicit intent:** Must consciously set variable AND specify tag +- **Command-line visibility:** Both appear in the command +- **Flexibility:** Supports both wipe-only and clean-install scenarios +- **Safety:** Very hard to accidentally trigger +- **Auditability:** Clear in logs what was intended + +#### Comparison with `never` Tag + +**Our approach:** +```yaml +when: web_app_wipe | bool +tags: + - web_app_wipe +``` + +**`never` tag approach:** +```yaml +tags: + - never + - web_app_wipe +``` + +**Differences:** + +| Aspect | Our Approach | `never` Tag | +|--------|-------------|-------------| +| **Flexibility** | Supports clean install scenario | Only wipe-only scenario | +| **Safety** | Double-gating (variable + tag) | Single-gating (tag only) | +| **Default behavior** | Skipped (when condition) | Skipped (never tag) | +| **Clean install** | `ansible-playbook deploy.yml -e "web_app_wipe=true"` | Not possible | +| **Wipe only** | `ansible-playbook deploy.yml -e "web_app_wipe=true" --tags web_app_wipe` | `ansible-playbook deploy.yml --tags never,web_app_wipe` | +| **Accidental execution** | Very difficult (needs both) | Moderate (needs tag) | + +**Why we chose variable + tag over `never`:** +- More flexible (supports clean install use case) +- Clearer intent (variable name is self-documenting) +- Better for production (explicit confirmation at two levels) +- Easier to understand (no special Ansible tag knowledge needed) + + +### 3.4 Test Results + +**Scenario 1: Normal deployment (wipe should NOT run)** +```bash +ansible-playbook playbooks/deploy.yml +PLAY [Deploy application] *************************************************************************************************************************************************** + +TASK [Gathering Facts] ****************************************************************************************************************************************************** +ok: [lab04-vm] + +TASK [docker : Install prerequisites for Docker] **************************************************************************************************************************** +ok: [lab04-vm] + +TASK [docker : Create directory for Docker GPG key] ************************************************************************************************************************* +ok: [lab04-vm] + +TASK [docker : Add Docker GPG key] ****************************************************************************************************************************************** +ok: [lab04-vm] + +TASK [docker : Get Ubuntu release codename] ********************************************************************************************************************************* +ok: [lab04-vm] + +TASK [docker : Get system architecture] ************************************************************************************************************************************* +ok: [lab04-vm] + +TASK [docker : Add Docker repository] *************************************************************************************************************************************** +ok: [lab04-vm] + +TASK [docker : Install Docker packages] ************************************************************************************************************************************* +ok: [lab04-vm] + +TASK [docker : Install python3-docker for Ansible docker modules] *********************************************************************************************************** +ok: [lab04-vm] + +TASK [docker : Ensure Docker service is enabled and started] **************************************************************************************************************** +ok: [lab04-vm] + +TASK [docker : Log Docker installation completion] ************************************************************************************************************************** +[WARNING]: Deprecation warnings can be disabled by setting `deprecation_warnings=False` in ansible.cfg. +[DEPRECATION WARNING]: INJECT_FACTS_AS_VARS default to `True` is deprecated, top-level facts will not be auto injected after the change. This feature will be removed from ansible-core version 2.24. +Origin: /Users/newspec/Desktop/DevOps/DevOps-Core-Course/ansible/roles/docker/tasks/main.yml:99:18 + +97 - name: Log Docker installation completion +98 copy: +99 content: "Docker installation completed at {{ ansible_date_time.iso8601 }}\n" + ^ column 18 + +Use `ansible_facts["fact_name"]` (no `ansible_` prefix) instead. + +changed: [lab04-vm] + +TASK [docker : Add user to docker group] ************************************************************************************************************************************ +ok: [lab04-vm] + +TASK [docker : Verify Docker installation] ********************************************************************************************************************************** +ok: [lab04-vm] + +TASK [docker : Display Docker version] ************************************************************************************************************************************** +ok: [lab04-vm] => { + "msg": "Docker version installed: Docker version 29.2.1, build a5c7197" +} + +TASK [docker : Log Docker configuration completion] ************************************************************************************************************************* +[DEPRECATION WARNING]: INJECT_FACTS_AS_VARS default to `True` is deprecated, top-level facts will not be auto injected after the change. This feature will be removed from ansible-core version 2.24. +Origin: /Users/newspec/Desktop/DevOps/DevOps-Core-Course/ansible/roles/docker/tasks/main.yml:129:18 + +127 - name: Log Docker configuration completion +128 copy: +129 content: "Docker configuration completed at {{ ansible_date_time.iso8601 }}\n" + ^ column 18 + +Use `ansible_facts["fact_name"]` (no `ansible_` prefix) instead. + +changed: [lab04-vm] + +TASK [web_app : Include wipe tasks] ***************************************************************************************************************************************** +included: /Users/newspec/Desktop/DevOps/DevOps-Core-Course/ansible/roles/web_app/tasks/wipe.yml for lab04-vm + +TASK [web_app : Check if docker-compose file exists] ************************************************************************************************************************ +skipping: [lab04-vm] + +TASK [web_app : Stop and remove containers with Docker Compose] ************************************************************************************************************* +skipping: [lab04-vm] + +TASK [web_app : Stop container manually if compose file doesn't exist] ****************************************************************************************************** +skipping: [lab04-vm] + +TASK [web_app : Remove docker-compose file] ********************************************************************************************************************************* +skipping: [lab04-vm] + +TASK [web_app : Remove application directory] ******************************************************************************************************************************* +skipping: [lab04-vm] + +TASK [web_app : Log wipe completion] **************************************************************************************************************************************** +skipping: [lab04-vm] + +TASK [web_app : Check if application is already running] ******************************************************************************************************************** +ok: [lab04-vm] + +TASK [web_app : Display current container status] *************************************************************************************************************************** +ok: [lab04-vm] => { + "msg": "Container devops-python is running" +} + +TASK [web_app : Create application directory] ******************************************************************************************************************************* +ok: [lab04-vm] + +TASK [web_app : Template docker-compose file] ******************************************************************************************************************************* +ok: [lab04-vm] + +TASK [web_app : Log in to Docker Hub] *************************************************************************************************************************************** +ok: [lab04-vm] + +TASK [web_app : Deploy with Docker Compose] ********************************************************************************************************************************* +ok: [lab04-vm] + +TASK [web_app : Display deployment result] ********************************************************************************************************************************** +ok: [lab04-vm] => { + "msg": "Deployment unchanged - containers are up to date" +} + +TASK [web_app : Wait for application port to be available (on target VM)] *************************************************************************************************** +ok: [lab04-vm] + +TASK [web_app : Verify health endpoint (from target VM)] ******************************************************************************************************************** +ok: [lab04-vm] + +TASK [web_app : Display health check result] ******************************************************************************************************************************** +ok: [lab04-vm] => { + "msg": "Application is healthy: {'status': 'healthy', 'timestamp': '2026-03-01T18:04:45.889812+00:00', 'uptime_seconds': 1099}" +} + +PLAY RECAP ****************************************************************************************************************************************************************** +lab04-vm : ok=26 changed=2 unreachable=0 failed=0 skipped=6 rescued=0 ignored=0 + +``` +```bash +ssh ubuntu@93.77.180.155 "docker ps" +CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES +1253f65bdeb0 newspec/python_app:latest "python app.py" 19 minutes ago Up 19 minutes 0.0.0.0:8000->8000/tcp, [::]:8000->8000/tcp devops-python +``` + Result: App deploys normally, wipe tasks skipped (tag not specified) + +**Scenario 2: Wipe only (remove existing deployment)** +```bash +ansible-playbook playbooks/deploy.yml \ + -e "web_app_wipe=true" \ + --tags web_app_wipe + PLAY [Deploy application] *************************************************************************************************************************************************** + +TASK [Gathering Facts] ****************************************************************************************************************************************************** +ok: [lab04-vm] + +TASK [web_app : Include wipe tasks] ***************************************************************************************************************************************** +included: /Users/newspec/Desktop/DevOps/DevOps-Core-Course/ansible/roles/web_app/tasks/wipe.yml for lab04-vm + +TASK [web_app : Check if docker-compose file exists] ************************************************************************************************************************ +ok: [lab04-vm] + +TASK [web_app : Stop and remove containers with Docker Compose] ************************************************************************************************************* +changed: [lab04-vm] + +TASK [web_app : Stop container manually if compose file doesn't exist] ****************************************************************************************************** +skipping: [lab04-vm] + +TASK [web_app : Remove docker-compose file] ********************************************************************************************************************************* +changed: [lab04-vm] + +TASK [web_app : Remove application directory] ******************************************************************************************************************************* +changed: [lab04-vm] + +TASK [web_app : Log wipe completion] **************************************************************************************************************************************** +ok: [lab04-vm] => { + "msg": "Application devops-python wiped successfully from /opt/devops-python" +} + +PLAY RECAP ****************************************************************************************************************************************************************** +lab04-vm : ok=7 changed=3 unreachable=0 failed=0 skipped=1 rescued=0 ignored=0 +``` +```bash +ssh ubuntu@93.77.180.155 "docker ps" +CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES +``` +```bash +ssh ubuntu@93.77.180.155 "ls /opt" +containerd +``` +Result: App removed, deployment skipped + +**Scenario 3: Clean reinstallation (wipe β†’ deploy)** +```bash +ansible-playbook playbooks/deploy.yml \ + -e "web_app_wipe=true" +PLAY [Deploy application] *************************************************************************************************************************************************** + +TASK [Gathering Facts] ****************************************************************************************************************************************************** +ok: [lab04-vm] + +TASK [docker : Install prerequisites for Docker] **************************************************************************************************************************** +ok: [lab04-vm] + +TASK [docker : Create directory for Docker GPG key] ************************************************************************************************************************* +ok: [lab04-vm] + +TASK [docker : Add Docker GPG key] ****************************************************************************************************************************************** +ok: [lab04-vm] + +TASK [docker : Get Ubuntu release codename] ********************************************************************************************************************************* +ok: [lab04-vm] + +TASK [docker : Get system architecture] ************************************************************************************************************************************* +ok: [lab04-vm] + +TASK [docker : Add Docker repository] *************************************************************************************************************************************** +ok: [lab04-vm] + +TASK [docker : Install Docker packages] ************************************************************************************************************************************* +ok: [lab04-vm] + +TASK [docker : Install python3-docker for Ansible docker modules] *********************************************************************************************************** +ok: [lab04-vm] + +TASK [docker : Ensure Docker service is enabled and started] **************************************************************************************************************** +ok: [lab04-vm] + +TASK [docker : Log Docker installation completion] ************************************************************************************************************************** +[WARNING]: Deprecation warnings can be disabled by setting `deprecation_warnings=False` in ansible.cfg. +[DEPRECATION WARNING]: INJECT_FACTS_AS_VARS default to `True` is deprecated, top-level facts will not be auto injected after the change. This feature will be removed from ansible-core version 2.24. +Origin: /Users/newspec/Desktop/DevOps/DevOps-Core-Course/ansible/roles/docker/tasks/main.yml:99:18 + +97 - name: Log Docker installation completion +98 copy: +99 content: "Docker installation completed at {{ ansible_date_time.iso8601 }}\n" + ^ column 18 + +Use `ansible_facts["fact_name"]` (no `ansible_` prefix) instead. + +changed: [lab04-vm] + +TASK [docker : Add user to docker group] ************************************************************************************************************************************ +ok: [lab04-vm] + +TASK [docker : Verify Docker installation] ********************************************************************************************************************************** +ok: [lab04-vm] + +TASK [docker : Display Docker version] ************************************************************************************************************************************** +ok: [lab04-vm] => { + "msg": "Docker version installed: Docker version 29.2.1, build a5c7197" +} + +TASK [docker : Log Docker configuration completion] ************************************************************************************************************************* +[DEPRECATION WARNING]: INJECT_FACTS_AS_VARS default to `True` is deprecated, top-level facts will not be auto injected after the change. This feature will be removed from ansible-core version 2.24. +Origin: /Users/newspec/Desktop/DevOps/DevOps-Core-Course/ansible/roles/docker/tasks/main.yml:129:18 + +127 - name: Log Docker configuration completion +128 copy: +129 content: "Docker configuration completed at {{ ansible_date_time.iso8601 }}\n" + ^ column 18 + +Use `ansible_facts["fact_name"]` (no `ansible_` prefix) instead. + +changed: [lab04-vm] + +TASK [web_app : Include wipe tasks] ***************************************************************************************************************************************** +included: /Users/newspec/Desktop/DevOps/DevOps-Core-Course/ansible/roles/web_app/tasks/wipe.yml for lab04-vm + +TASK [web_app : Check if docker-compose file exists] ************************************************************************************************************************ +ok: [lab04-vm] + +TASK [web_app : Stop and remove containers with Docker Compose] ************************************************************************************************************* +skipping: [lab04-vm] + +TASK [web_app : Stop container manually if compose file doesn't exist] ****************************************************************************************************** +ok: [lab04-vm] + +TASK [web_app : Remove docker-compose file] ********************************************************************************************************************************* +ok: [lab04-vm] + +TASK [web_app : Remove application directory] ******************************************************************************************************************************* +ok: [lab04-vm] + +TASK [web_app : Log wipe completion] **************************************************************************************************************************************** +ok: [lab04-vm] => { + "msg": "Application devops-python wiped successfully from /opt/devops-python" +} + +TASK [web_app : Check if application is already running] ******************************************************************************************************************** +ok: [lab04-vm] + +TASK [web_app : Display current container status] *************************************************************************************************************************** +ok: [lab04-vm] => { + "msg": "Container devops-python is not running" +} + +TASK [web_app : Create application directory] ******************************************************************************************************************************* +changed: [lab04-vm] + +TASK [web_app : Template docker-compose file] ******************************************************************************************************************************* +changed: [lab04-vm] + +TASK [web_app : Log in to Docker Hub] *************************************************************************************************************************************** +ok: [lab04-vm] + +TASK [web_app : Deploy with Docker Compose] ********************************************************************************************************************************* +changed: [lab04-vm] + +TASK [web_app : Display deployment result] ********************************************************************************************************************************** +ok: [lab04-vm] => { + "msg": "Deployment changed - containers are updated" +} + +TASK [web_app : Wait for application port to be available (on target VM)] *************************************************************************************************** +ok: [lab04-vm] + +TASK [web_app : Verify health endpoint (from target VM)] ******************************************************************************************************************** +ok: [lab04-vm] + +TASK [web_app : Display health check result] ******************************************************************************************************************************** +ok: [lab04-vm] => { + "msg": "Application is healthy: {'status': 'healthy', 'timestamp': '2026-03-01T18:20:21.216078+00:00', 'uptime_seconds': 6}" +} + +PLAY RECAP ****************************************************************************************************************************************************************** +lab04-vm : ok=31 changed=5 unreachable=0 failed=0 skipped=1 rescued=0 ignored=0 +``` +```bash +ssh ubuntu@93.77.180.155 "docker ps" +CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES +5f5c4479353f newspec/python_app:latest "python app.py" 29 seconds ago Up 28 seconds 0.0.0.0:8000->8000/tcp, [::]:8000->8000/tcp devops-python +``` +Result: Old app removed, new app deployed (clean reinstall) + +**Scenario 4a: Safety check - Tag specified but variable false** +```bash +ansible-playbook playbooks/deploy.yml --tags web_app_wipe +PLAY [Deploy application] *************************************************************************************************************************************************** + +TASK [Gathering Facts] ****************************************************************************************************************************************************** +ok: [lab04-vm] + +TASK [web_app : Include wipe tasks] ***************************************************************************************************************************************** +included: /Users/newspec/Desktop/DevOps/DevOps-Core-Course/ansible/roles/web_app/tasks/wipe.yml for lab04-vm + +TASK [web_app : Check if docker-compose file exists] ************************************************************************************************************************ +skipping: [lab04-vm] + +TASK [web_app : Stop and remove containers with Docker Compose] ************************************************************************************************************* +skipping: [lab04-vm] + +TASK [web_app : Stop container manually if compose file doesn't exist] ****************************************************************************************************** +skipping: [lab04-vm] + +TASK [web_app : Remove docker-compose file] ********************************************************************************************************************************* +skipping: [lab04-vm] + +TASK [web_app : Remove application directory] ******************************************************************************************************************************* +skipping: [lab04-vm] + +TASK [web_app : Log wipe completion] **************************************************************************************************************************************** +skipping: [lab04-vm] + +PLAY RECAP ****************************************************************************************************************************************************************** +lab04-vm : ok=2 changed=0 unreachable=0 failed=0 skipped=6 rescued=0 ignored=0 +``` +Result: Wipe tasks skipped (when condition blocks it), deployment runs normally + +**Scenario 4b: Safety check - Variable true, deployment skipped** +```bash +ansible-playbook playbooks/deploy.yml \ + -e "web_app_wipe=true" \ + --tags web_app_wipe +PLAY [Deploy application] *************************************************************************************************************************************************** + +TASK [Gathering Facts] ****************************************************************************************************************************************************** +ok: [lab04-vm] + +TASK [web_app : Include wipe tasks] ***************************************************************************************************************************************** +included: /Users/newspec/Desktop/DevOps/DevOps-Core-Course/ansible/roles/web_app/tasks/wipe.yml for lab04-vm + +TASK [web_app : Check if docker-compose file exists] ************************************************************************************************************************ +ok: [lab04-vm] + +TASK [web_app : Stop and remove containers with Docker Compose] ************************************************************************************************************* +changed: [lab04-vm] + +TASK [web_app : Stop container manually if compose file doesn't exist] ****************************************************************************************************** +skipping: [lab04-vm] + +TASK [web_app : Remove docker-compose file] ********************************************************************************************************************************* +changed: [lab04-vm] + +TASK [web_app : Remove application directory] ******************************************************************************************************************************* +changed: [lab04-vm] + +TASK [web_app : Log wipe completion] **************************************************************************************************************************************** +ok: [lab04-vm] => { + "msg": "Application devops-python wiped successfully from /opt/devops-python" +} + +PLAY RECAP ****************************************************************************************************************************************************************** +lab04-vm : ok=7 changed=3 unreachable=0 failed=0 skipped=1 rescued=0 ignored=0 +``` +Result: Only wipe runs, no deployment + +#### Screenshot of application running after clean reinstall +![alt text](image.png) + +### 3.5 Research Questions Answered + +**1. Why use both variable AND tag?** +A: Double safety mechanism prevents accidental data loss: +- Variable alone: Could be set accidentally in vars file +- Tag alone: Could be run without realizing consequences +- Both together: Requires explicit, conscious decision + +**2. What's the difference between `never` tag and this approach?** +A: +- `never` tag: Tasks NEVER run unless explicitly called with `--tags never` +- Our approach: Tasks can run in two scenarios (wipe-only OR clean-install) +- Our approach is more flexible for the clean reinstall use case + +**3. Why must wipe logic come BEFORE deployment in main.yml?** +A: To support the clean reinstall scenario where we want to: +1. First wipe the old installation +2. Then deploy the new installation +If wipe came after, we'd deploy then immediately wipe! + +**4. When would you want clean reinstallation vs. rolling update?** +A: +- Clean reinstall: Major version changes, corrupted state, testing from scratch +- Rolling update: Minor updates, zero-downtime requirements, production environments + +**5. How would you extend this to wipe Docker images and volumes too?** +A: Add tasks to wipe.yml: +```yaml +- name: Remove Docker images + community.docker.docker_image: + name: "{{ docker_image }}" + tag: "{{ docker_tag }}" + state: absent + +- name: Remove Docker volumes + community.docker.docker_volume: + name: "{{ web_app_name }}_data" + state: absent +``` + +--- + +## Task 4: CI/CD Integration + +### 4.1 Workflow Architecture + +**File:** [`.github/workflows/ansible-deploy.yml`](../../.github/workflows/ansible-deploy.yml) + +**CI/CD Flow:** +``` +Code Push β†’ Lint Ansible β†’ Deploy with Ansible β†’ Verify Deployment +``` + +**Workflow Triggers:** +- Push to `main`, `master`, or `lab06` branches (when Ansible files change) +- Pull requests to `main`, `master`, or `lab06` branches +- Manual trigger via `workflow_dispatch` + +**Benefits:** +- **Consistency:** Same process every time +- **Speed:** Automatic deployments on push +- **Safety:** Linting catches errors before execution +- **Auditability:** GitHub logs every deployment +- **Integration:** Combines with testing, building, scanning + +### 4.2 Setup Steps + +#### Step 1: Create GitHub Repository Secrets + +Navigate to your GitHub repository β†’ Settings β†’ Secrets and variables β†’ Actions β†’ New repository secret + +**Required Secrets (mandatory):** + +1. **ANSIBLE_VAULT_PASSWORD** + - Value: Your Ansible Vault password + - Used to decrypt encrypted variables in `group_vars/all.yml` + - Example: `your_vault_password_here` + - **Status:** Validated in "Create Vault password file" step + +2. **SSH_PRIVATE_KEY** + - Value: Private SSH key for accessing target VM + - Generate with: `ssh-keygen -t ed25519 -C "github-actions"` + - Copy private key: `cat ~/.ssh/id_ed25519` + - Add public key to VM: `ssh-copy-id -i ~/.ssh/id_ed25519.pub ubuntu@vm_ip` + - **Status:** Validated in "Setup SSH" step + +3. **VM_HOST** + - Value: Target VM IP address or hostname + - Example: `93.77.180.155` + - **Status:** Validated in "Setup SSH" step + +**Optional Secrets:** + +4. **DOCKERHUB_USERNAME** (if using private images) + - Value: Your Docker Hub username + - Example: `newspec` + - **Note:** Stored in encrypted `group_vars/all.yml` + +5. **DOCKERHUB_PASSWORD** (if using private images) + - Value: Your Docker Hub password or access token + - Recommended: Use access token instead of password + - **Note:** Stored in encrypted `group_vars/all.yml` + +#### Step 2: Create Workflow File + +**File:** `.github/workflows/ansible-deploy.yml` + +```yaml +name: Ansible Deployment - Python App + +on: + push: + branches: [ main, master, lab06 ] + paths: + - 'ansible/vars/app_python.yml' + - 'ansible/playbooks/deploy_python.yml' + - 'ansible/playbooks/deploy.yml' + - 'ansible/roles/web_app/**' + - 'ansible/roles/common/**' + - 'ansible/roles/docker/**' + - '.github/workflows/ansible-deploy.yml' + pull_request: + branches: [ main, master, lab06 ] + paths: + - 'ansible/vars/app_python.yml' + - 'ansible/playbooks/deploy_python.yml' + - 'ansible/roles/web_app/**' + workflow_dispatch: + +jobs: + lint: + name: Ansible Lint - Python App + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.12' + + - name: Install dependencies + run: | + pip install ansible ansible-lint + + - name: Run ansible-lint + run: | + cd ansible + ansible-lint playbooks/deploy_python.yml playbooks/deploy.yml + + deploy: + name: Deploy Python Application + needs: lint + runs-on: ubuntu-latest + if: github.event_name == 'push' || github.event_name == 'workflow_dispatch' + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.12' + + - name: Install Ansible and dependencies + run: | + pip install ansible + ansible-galaxy collection install community.docker + + - name: Setup SSH + run: | + if [ -z "${{ secrets.SSH_PRIVATE_KEY }}" ]; then + echo "Error: SSH_PRIVATE_KEY secret is not set" + echo "Please configure the SSH_PRIVATE_KEY secret in repository settings" + exit 1 + fi + if [ -z "${{ secrets.VM_HOST }}" ]; then + echo "Error: VM_HOST secret is not set" + echo "Please configure the VM_HOST secret in repository settings" + exit 1 + fi + + mkdir -p ~/.ssh + echo "${{ secrets.SSH_PRIVATE_KEY }}" > ~/.ssh/yandex_cloud_key + chmod 600 ~/.ssh/yandex_cloud_key + ssh-keyscan -H ${{ secrets.VM_HOST }} >> ~/.ssh/known_hosts + + - name: Create Vault password file + env: + ANSIBLE_VAULT_PASSWORD: ${{ secrets.ANSIBLE_VAULT_PASSWORD }} + run: | + if [ -z "$ANSIBLE_VAULT_PASSWORD" ]; then + echo "Error: ANSIBLE_VAULT_PASSWORD secret is not set" + echo "Please configure the ANSIBLE_VAULT_PASSWORD secret in repository settings" + exit 1 + fi + echo "$ANSIBLE_VAULT_PASSWORD" > /tmp/vault_pass + chmod 600 /tmp/vault_pass + + - name: Deploy Python App with Ansible + run: | + cd ansible + ansible-playbook playbooks/deploy_python.yml \ + --vault-password-file /tmp/vault_pass \ + --tags "app_deploy" \ + -e @group_vars/all.yml + + - name: Cleanup sensitive files + if: always() + run: | + rm -f /tmp/vault_pass + rm -f ~/.ssh/yandex_cloud_key + + - name: Verify Python App Deployment + run: | + echo "Waiting for application to start..." + sleep 10 + + echo "Testing main endpoint..." + if ! curl -f http://${{ secrets.VM_HOST }}:8000; then + echo "Error: Python app is not accessible at http://${{ secrets.VM_HOST }}:8000" + exit 1 + fi + + echo "Testing health endpoint..." + if ! curl -f http://${{ secrets.VM_HOST }}:8000/health; then + echo "Error: Python app health check failed at http://${{ secrets.VM_HOST }}:8000/health" + exit 1 + fi + + echo "Deployment verification successful!" +``` + +**Key Features:** +- **Specific Path Triggers:** Only runs when relevant Ansible files change +- **Python App Focus:** Deploys using `deploy_python.yml` playbook +- **Mandatory Secrets Validation:** Validates all required secrets (SSH_PRIVATE_KEY, VM_HOST, ANSIBLE_VAULT_PASSWORD) before execution with clear error messages +- **Early Failure Detection:** Fails fast if secrets are missing, preventing confusing SSH/Ansible errors +- **Vault Integration:** Securely handles Ansible Vault password via environment variable +- **Secure Cleanup:** Uses `if: always()` to ensure sensitive files (vault password and SSH key) are always deleted +- **Comprehensive Verification:** Tests both main endpoint and health check +- **Clean Deployment:** Uses `--tags "app_deploy"` for targeted deployment + +#### Step 3: Verify Playbook Configuration + +**File:** `ansible/playbooks/deploy_python.yml` + +Ensure your deployment playbook is properly configured: + +```yaml +--- +- name: Deploy Python Application + hosts: webservers + become: false + vars_files: + - ../vars/app_python.yml + + roles: + - role: web_app + tags: app_deploy +``` + +**Key Points:** +- Uses `vars_files` to load Python app configuration +- Tags role with `app_deploy` for selective execution +- Relies on inventory configuration from `inventory/hosts.ini` + +#### Step 4: Commit and Push + +```bash +git add .github/workflows/ansible-deploy.yml +git add ansible/ +git commit -m "Add CI/CD workflow for Python app deployment" +git push origin main +``` + +#### Step 5: Monitor Workflow Execution + +1. Go to GitHub repository β†’ Actions tab +2. Click on the running workflow +3. Monitor each job (lint, deploy) +4. Check logs for any errors +5. Verify deployment success + +#### Step 6: Add Status Badge to README + +**File:** `README.md` or `ansible/README.md` + +```markdown +# DevOps Core Course + +[![Ansible Deployment](https://github.com/YOUR_USERNAME/YOUR_REPO/actions/workflows/ansible-deploy.yml/badge.svg)](https://github.com/YOUR_USERNAME/YOUR_REPO/actions/workflows/ansible-deploy.yml) + +## Ansible Automation + +Automated deployment with GitHub Actions... +``` + +Replace `YOUR_USERNAME` and `YOUR_REPO` with your actual values. + +#### Step 7: Troubleshooting Common Issues + +**Issue 1: Missing Required Secrets** +``` +Error: SSH_PRIVATE_KEY secret is not set +Error: VM_HOST secret is not set +Error: ANSIBLE_VAULT_PASSWORD secret is not set +``` +**Solution:** All three secrets are mandatory. Add them in GitHub repository settings β†’ Secrets and variables β†’ Actions + +**Validation happens early:** +- `SSH_PRIVATE_KEY` and `VM_HOST` validated in "Setup SSH" step +- `ANSIBLE_VAULT_PASSWORD` validated in "Create Vault password file" step + +**Issue 2: SSH Connection Failed** +```bash +# Solution: Verify SSH key is correct and added to GitHub Secrets +ssh -i ~/.ssh/yandex_cloud_key ubuntu@vm_ip + +# Check known_hosts +ssh-keyscan -H vm_ip +``` + +**Issue 3: Vault Decryption Failed** +```bash +# Solution: Verify vault password secret matches your local vault password +ansible-vault view group_vars/all.yml --vault-password-file <(echo "password") +``` + +**Issue 4: Ansible Module Not Found** +```bash +# Solution: Ensure community.docker collection is installed (workflow does this automatically) +ansible-galaxy collection install community.docker +``` + +**Issue 5: Undefined Variable Error** +``` +'dockerhub_username' is undefined +``` +**Solution:** Ensure `-e @group_vars/all.yml` is included in ansible-playbook command + +#### Step 8: Test Deployment + +```bash +# Make a small change to trigger workflow +echo "# Test change" >> ansible/vars/app_python.yml +git add ansible/vars/app_python.yml +git commit -m "Test CI/CD workflow for Python app" +git push origin main + +# Watch Actions tab for workflow execution +# Workflow will only trigger if Python app-related files are changed +``` + +**Alternative: Manual Trigger** +```bash +# Trigger workflow manually via GitHub UI: +# 1. Go to Actions tab +# 2. Select "Ansible Deployment - Python App" workflow +# 3. Click "Run workflow" button +# 4. Select branch and click "Run workflow" +``` + +### 4.3 Evidence of Automated Deployments + +#### Screenshot of successful workflow run +![alt text](image-1.png) + +#### Output logs showing ansible-lint passing +``` +Run cd ansible +Passed: 0 failure(s), 0 warning(s) in 7 files processed of 7 encountered. Last profile that met the validation criteria was 'production'. +``` + +#### Output logs showing ansible-playbook execution +``` +Run cd ansible + +PLAY [Deploy Python Application] *********************************************** + +TASK [Gathering Facts] ********************************************************* +ok: [lab04-vm] + +TASK [web_app : Check if application is already running] *********************** +ok: [lab04-vm] + +TASK [web_app : Display current container status] ****************************** +ok: [lab04-vm] => { + "msg": "Container devops-python is running" +} + +TASK [web_app : Create application directory] ********************************** +ok: [lab04-vm] + +TASK [web_app : Template docker-compose file] ********************************** +changed: [lab04-vm] + +TASK [web_app : Log in to Docker Hub] ****************************************** +ok: [lab04-vm] + +TASK [web_app : Deploy with Docker Compose] ************************************ +ok: [lab04-vm] + +TASK [web_app : Display deployment result] ************************************* +ok: [lab04-vm] => { + "msg": "Deployment unchanged - containers are up to date" +} + +TASK [web_app : Wait for application port to be available (on target VM)] ****** +ok: [lab04-vm] + +TASK [web_app : Verify health endpoint (from target VM)] *********************** +ok: [lab04-vm] + +TASK [web_app : Display health check result] *********************************** +ok: [lab04-vm] => { + "msg": "Application is healthy: {'status': 'healthy', 'timestamp': '2026-03-01T19:30:19.234174+00:00', 'uptime_seconds': 3801}" +} + +PLAY RECAP ********************************************************************* +lab04-vm : ok=11 changed=1 unreachable=0 failed=0 skipped=0 rescued=0 ignored=0 +``` +#### Verification step output showing app responding +``` +Run if [ -n "***" ]; then + % Total % Received % Xferd Average Speed Time Time Time Current + Dload Upload Total Spent Left Speed + + 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 +100 699 100 699 0 0 1891 0 --:--:-- --:--:-- --:--:-- 1889 +100 699 100 699 0 0 1891 0 --:--:-- --:--:-- --:--:-- 1889 + % Total % Received % Xferd Average Speed Time Time Time Current + Dload Upload Total Spent Left Speed + + 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 +100 89 100 89 0 0 235 0 --:--:-- --:--:-- --:--:-- 236 +{"service":{"name":"devops-info-service","version":"1.0.0","description":"DevOps course info service","framework":"FastAPI"},"system":{"hostname":"01f6effa0e13","platform":"Linux","platform_version":"#100-Ubuntu SMP PREEMPT_DYNAMIC Tue Jan 13 16:40:06 UTC 2026","architecture":"x86_64","cpu_count":2,"python_version":"3.12.12"},"runtime":{"uptime_seconds":3812,"uptime_human":"1 hours, 3 minutes","current_time":"2026-03-01T19:30:30.146722+00:00","timezone":"UTC"},"re***uest":{"client_ip":"52.159.247.196","user_agent":"curl/8.5.0","method":"GET","path":"/"},"endpoints":[{"path":"/","method":"GET","description":"Service information"},{"path":"/health","method":"GET","description":"Health check"}]}{"status":"healthy","timestamp":"2026-03-01T19:30:30.531667+00:00","uptime_seconds":3812} +``` +#### Status badge in README showing passing +[![Ansible Deployment - Python App](https://github.com/newspec/DevOps-Core-Course/actions/workflows/ansible-deploy.yml/badge.svg)](https://github.com/newspec/DevOps-Core-Course/actions/workflows/ansible-deploy.yml) + +### 4.8 Research Questions Answered + +**1. What are the security implications of storing SSH keys in GitHub Secrets?** +A: +- **Pros:** Encrypted at rest, access-controlled, audit logged +- **Cons:** GitHub has access, potential for compromise if GitHub is breached +- **Best Practice:** Use dedicated deployment keys with minimal permissions, rotate regularly +- **Alternative:** Use GitHub's OIDC for keyless authentication + +**2. How would you implement a staging β†’ production deployment pipeline?** +A: +```yaml +jobs: + deploy-staging: + # Deploy to staging + + manual-approval: + needs: deploy-staging + environment: production # Requires manual approval + + deploy-production: + needs: manual-approval + # Deploy to production +``` + +**3. What would you add to make rollbacks possible?** +A: +- Tag Docker images with git commit SHA +- Store previous deployment state +- Create rollback playbook that deploys previous version +- Add workflow_dispatch input for version selection +- Keep deployment history in artifact storage + +**4. How does self-hosted runner improve security compared to GitHub-hosted?** +A: +- No SSH keys needed (runner is on target network) +- Secrets never leave your infrastructure +- Full control over runner environment +- Can use internal DNS/networks +- Reduced attack surface (no internet-exposed SSH) + +--- + +# Testing Results +```bash +# Test provision with only docker +ansible-playbook playbooks/provision.yml --tags "docker" +PLAY [Provision web servers] ************************************************************************************************************************************************ + +TASK [Gathering Facts] ****************************************************************************************************************************************************** +ok: [lab04-vm] + +TASK [docker : Install prerequisites for Docker] **************************************************************************************************************************** +ok: [lab04-vm] + +TASK [docker : Create directory for Docker GPG key] ************************************************************************************************************************* +ok: [lab04-vm] + +TASK [docker : Add Docker GPG key] ****************************************************************************************************************************************** +ok: [lab04-vm] + +TASK [docker : Get Ubuntu release codename] ********************************************************************************************************************************* +ok: [lab04-vm] + +TASK [docker : Get system architecture] ************************************************************************************************************************************* +ok: [lab04-vm] + +TASK [docker : Add Docker repository] *************************************************************************************************************************************** +ok: [lab04-vm] + +TASK [docker : Install Docker packages] ************************************************************************************************************************************* +ok: [lab04-vm] + +TASK [docker : Install python3-docker for Ansible docker modules] *********************************************************************************************************** +ok: [lab04-vm] + +TASK [docker : Ensure Docker service is enabled and started] **************************************************************************************************************** +ok: [lab04-vm] + +TASK [docker : Log Docker installation completion] ************************************************************************************************************************** +[WARNING]: Deprecation warnings can be disabled by setting `deprecation_warnings=False` in ansible.cfg. +[DEPRECATION WARNING]: INJECT_FACTS_AS_VARS default to `True` is deprecated, top-level facts will not be auto injected after the change. This feature will be removed from ansible-core version 2.24. +Origin: /Users/newspec/Desktop/DevOps/DevOps-Core-Course/ansible/roles/docker/tasks/main.yml:99:18 + +97 - name: Log Docker installation completion +98 copy: +99 content: "Docker installation completed at {{ ansible_date_time.iso8601 }}\n" + ^ column 18 + +Use `ansible_facts["fact_name"]` (no `ansible_` prefix) instead. + +changed: [lab04-vm] + +TASK [docker : Add user to docker group] ************************************************************************************************************************************ +ok: [lab04-vm] + +TASK [docker : Verify Docker installation] ********************************************************************************************************************************** +ok: [lab04-vm] + +TASK [docker : Display Docker version] ************************************************************************************************************************************** +ok: [lab04-vm] => { + "msg": "Docker version installed: Docker version 29.2.1, build a5c7197" +} + +TASK [docker : Log Docker configuration completion] ************************************************************************************************************************* +[DEPRECATION WARNING]: INJECT_FACTS_AS_VARS default to `True` is deprecated, top-level facts will not be auto injected after the change. This feature will be removed from ansible-core version 2.24. +Origin: /Users/newspec/Desktop/DevOps/DevOps-Core-Course/ansible/roles/docker/tasks/main.yml:129:18 + +127 - name: Log Docker configuration completion +128 copy: +129 content: "Docker configuration completed at {{ ansible_date_time.iso8601 }}\n" + ^ column 18 + +Use `ansible_facts["fact_name"]` (no `ansible_` prefix) instead. + +changed: [lab04-vm] + +PLAY RECAP ****************************************************************************************************************************************************************** +lab04-vm : ok=15 changed=2 unreachable=0 failed=0 skipped=0 rescued=0 ignored=0 +``` +```bash +# Skip common role +ansible-playbook playbooks/provision.yml --skip-tags "common" +PLAY [Provision web servers] ************************************************************************************************************************************************ + +TASK [Gathering Facts] ****************************************************************************************************************************************************** +ok: [lab04-vm] + +TASK [docker : Install prerequisites for Docker] **************************************************************************************************************************** +ok: [lab04-vm] + +TASK [docker : Create directory for Docker GPG key] ************************************************************************************************************************* +ok: [lab04-vm] + +TASK [docker : Add Docker GPG key] ****************************************************************************************************************************************** +ok: [lab04-vm] + +TASK [docker : Get Ubuntu release codename] ********************************************************************************************************************************* +ok: [lab04-vm] + +TASK [docker : Get system architecture] ************************************************************************************************************************************* +ok: [lab04-vm] + +TASK [docker : Add Docker repository] *************************************************************************************************************************************** +ok: [lab04-vm] + +TASK [docker : Install Docker packages] ************************************************************************************************************************************* +ok: [lab04-vm] + +TASK [docker : Install python3-docker for Ansible docker modules] *********************************************************************************************************** +ok: [lab04-vm] + +TASK [docker : Ensure Docker service is enabled and started] **************************************************************************************************************** +ok: [lab04-vm] + +TASK [docker : Log Docker installation completion] ************************************************************************************************************************** +[WARNING]: Deprecation warnings can be disabled by setting `deprecation_warnings=False` in ansible.cfg. +[DEPRECATION WARNING]: INJECT_FACTS_AS_VARS default to `True` is deprecated, top-level facts will not be auto injected after the change. This feature will be removed from ansible-core version 2.24. +Origin: /Users/newspec/Desktop/DevOps/DevOps-Core-Course/ansible/roles/docker/tasks/main.yml:99:18 + +97 - name: Log Docker installation completion +98 copy: +99 content: "Docker installation completed at {{ ansible_date_time.iso8601 }}\n" + ^ column 18 + +Use `ansible_facts["fact_name"]` (no `ansible_` prefix) instead. + +changed: [lab04-vm] + +TASK [docker : Add user to docker group] ************************************************************************************************************************************ +ok: [lab04-vm] + +TASK [docker : Verify Docker installation] ********************************************************************************************************************************** +ok: [lab04-vm] + +TASK [docker : Display Docker version] ************************************************************************************************************************************** +ok: [lab04-vm] => { + "msg": "Docker version installed: Docker version 29.2.1, build a5c7197" +} + +TASK [docker : Log Docker configuration completion] ************************************************************************************************************************* +[DEPRECATION WARNING]: INJECT_FACTS_AS_VARS default to `True` is deprecated, top-level facts will not be auto injected after the change. This feature will be removed from ansible-core version 2.24. +Origin: /Users/newspec/Desktop/DevOps/DevOps-Core-Course/ansible/roles/docker/tasks/main.yml:129:18 + +127 - name: Log Docker configuration completion +128 copy: +129 content: "Docker configuration completed at {{ ansible_date_time.iso8601 }}\n" + ^ column 18 + +Use `ansible_facts["fact_name"]` (no `ansible_` prefix) instead. + +changed: [lab04-vm] + +PLAY RECAP ****************************************************************************************************************************************************************** +lab04-vm : ok=15 changed=2 unreachable=0 failed=0 skipped=0 rescued=0 ignored=0 +``` +```bash +# Install packages only across all roles +ansible-playbook playbooks/provision.yml --tags "packages" +PLAY [Provision web servers] ************************************************************************************************************************************************ + +TASK [Gathering Facts] ****************************************************************************************************************************************************** +ok: [lab04-vm] + +TASK [common : Update apt cache] ******************************************************************************************************************************************** +ok: [lab04-vm] + +TASK [common : Install common packages] ************************************************************************************************************************************* +ok: [lab04-vm] + +TASK [common : Log package installation completion] ************************************************************************************************************************* +[WARNING]: Deprecation warnings can be disabled by setting `deprecation_warnings=False` in ansible.cfg. +[DEPRECATION WARNING]: INJECT_FACTS_AS_VARS default to `True` is deprecated, top-level facts will not be auto injected after the change. This feature will be removed from ansible-core version 2.24. +Origin: /Users/newspec/Desktop/DevOps/DevOps-Core-Course/ansible/roles/common/tasks/main.yml:36:18 + +34 - name: Log package installation completion +35 copy: +36 content: "Common packages installation completed at {{ ansible_date_time.iso8601 }}\n" + ^ column 18 + +Use `ansible_facts["fact_name"]` (no `ansible_` prefix) instead. + +changed: [lab04-vm] + +PLAY RECAP ****************************************************************************************************************************************************************** +lab04-vm : ok=4 changed=1 unreachable=0 failed=0 skipped=0 rescued=0 ignored=0 +``` +```bash +# Check mode to see what would run +ansible-playbook playbooks/provision.yml --tags "docker" --check +PLAY [Provision web servers] ************************************************************************************************************************************************ + +TASK [Gathering Facts] ****************************************************************************************************************************************************** +ok: [lab04-vm] + +TASK [docker : Install prerequisites for Docker] **************************************************************************************************************************** +ok: [lab04-vm] + +TASK [docker : Create directory for Docker GPG key] ************************************************************************************************************************* +ok: [lab04-vm] + +TASK [docker : Add Docker GPG key] ****************************************************************************************************************************************** +ok: [lab04-vm] + +TASK [docker : Get Ubuntu release codename] ********************************************************************************************************************************* +skipping: [lab04-vm] + +TASK [docker : Get system architecture] ************************************************************************************************************************************* +skipping: [lab04-vm] + +TASK [docker : Add Docker repository] *************************************************************************************************************************************** +changed: [lab04-vm] + +TASK [docker : Install Docker packages] ************************************************************************************************************************************* +ok: [lab04-vm] + +TASK [docker : Install python3-docker for Ansible docker modules] *********************************************************************************************************** +ok: [lab04-vm] + +TASK [docker : Ensure Docker service is enabled and started] **************************************************************************************************************** +ok: [lab04-vm] + +TASK [docker : Log Docker installation completion] ************************************************************************************************************************** +[WARNING]: Deprecation warnings can be disabled by setting `deprecation_warnings=False` in ansible.cfg. +[DEPRECATION WARNING]: INJECT_FACTS_AS_VARS default to `True` is deprecated, top-level facts will not be auto injected after the change. This feature will be removed from ansible-core version 2.24. +Origin: /Users/newspec/Desktop/DevOps/DevOps-Core-Course/ansible/roles/docker/tasks/main.yml:99:18 + +97 - name: Log Docker installation completion +98 copy: +99 content: "Docker installation completed at {{ ansible_date_time.iso8601 }}\n" + ^ column 18 + +Use `ansible_facts["fact_name"]` (no `ansible_` prefix) instead. + +changed: [lab04-vm] + +TASK [docker : Add user to docker group] ************************************************************************************************************************************ +ok: [lab04-vm] + +TASK [docker : Verify Docker installation] ********************************************************************************************************************************** +skipping: [lab04-vm] + +TASK [docker : Display Docker version] ************************************************************************************************************************************** +ok: [lab04-vm] => { + "msg": "Docker version installed: " +} + +TASK [docker : Log Docker configuration completion] ************************************************************************************************************************* +[DEPRECATION WARNING]: INJECT_FACTS_AS_VARS default to `True` is deprecated, top-level facts will not be auto injected after the change. This feature will be removed from ansible-core version 2.24. +Origin: /Users/newspec/Desktop/DevOps/DevOps-Core-Course/ansible/roles/docker/tasks/main.yml:129:18 + +127 - name: Log Docker configuration completion +128 copy: +129 content: "Docker configuration completed at {{ ansible_date_time.iso8601 }}\n" + ^ column 18 + +Use `ansible_facts["fact_name"]` (no `ansible_` prefix) instead. + +changed: [lab04-vm] + +RUNNING HANDLER [docker : restart docker] *********************************************************************************************************************************** +changed: [lab04-vm] + +PLAY RECAP ****************************************************************************************************************************************************************** +lab04-vm : ok=13 changed=4 unreachable=0 failed=0 skipped=3 rescued=0 ignored=0 +``` +```bash +# Run only docker installation tasks +ansible-playbook playbooks/provision.yml --tags "docker_install" +PLAY [Provision web servers] ************************************************************************************************************************************************ + +TASK [Gathering Facts] ****************************************************************************************************************************************************** +ok: [lab04-vm] + +TASK [docker : Install prerequisites for Docker] **************************************************************************************************************************** +ok: [lab04-vm] + +TASK [docker : Create directory for Docker GPG key] ************************************************************************************************************************* +ok: [lab04-vm] + +TASK [docker : Add Docker GPG key] ****************************************************************************************************************************************** +ok: [lab04-vm] + +TASK [docker : Get Ubuntu release codename] ********************************************************************************************************************************* +ok: [lab04-vm] + +TASK [docker : Get system architecture] ************************************************************************************************************************************* +ok: [lab04-vm] + +TASK [docker : Add Docker repository] *************************************************************************************************************************************** +ok: [lab04-vm] + +TASK [docker : Install Docker packages] ************************************************************************************************************************************* +ok: [lab04-vm] + +TASK [docker : Install python3-docker for Ansible docker modules] *********************************************************************************************************** +ok: [lab04-vm] + +TASK [docker : Ensure Docker service is enabled and started] **************************************************************************************************************** +ok: [lab04-vm] + +TASK [docker : Log Docker installation completion] ************************************************************************************************************************** +[WARNING]: Deprecation warnings can be disabled by setting `deprecation_warnings=False` in ansible.cfg. +[DEPRECATION WARNING]: INJECT_FACTS_AS_VARS default to `True` is deprecated, top-level facts will not be auto injected after the change. This feature will be removed from ansible-core version 2.24. +Origin: /Users/newspec/Desktop/DevOps/DevOps-Core-Course/ansible/roles/docker/tasks/main.yml:99:18 + +97 - name: Log Docker installation completion +98 copy: +99 content: "Docker installation completed at {{ ansible_date_time.iso8601 }}\n" + ^ column 18 + +Use `ansible_facts["fact_name"]` (no `ansible_` prefix) instead. + +changed: [lab04-vm] + +PLAY RECAP ****************************************************************************************************************************************************************** +lab04-vm : ok=11 changed=1 unreachable=0 failed=0 skipped=0 rescued=0 ignored=0 +``` + +### 1.4 Output showing error handling with rescue block triggered +```bash +ansible-playbook playbooks/test_rescue.yml + LAY [Test rescue block in common role] *********************************************************************************************************************** + +TASK [Gathering Facts] **************************************************************************************************************************************** +ok: [lab04-vm] + +TASK [Backup original sources.list] *************************************************************************************************************************** +changed: [lab04-vm] + +TASK [Add invalid repository to trigger error] **************************************************************************************************************** +changed: [lab04-vm] + +TASK [Try to update apt cache (will fail)] ******************************************************************************************************************** +[WARNING]: Failed to update cache after 1 retries due to , retrying +[WARNING]: Sleeping for 1 seconds, before attempting to refresh the cache again +[WARNING]: Failed to update cache after 2 retries due to , retrying +[WARNING]: Sleeping for 2 seconds, before attempting to refresh the cache again +[WARNING]: Failed to update cache after 3 retries due to , retrying +[WARNING]: Sleeping for 4 seconds, before attempting to refresh the cache again +[WARNING]: Failed to update cache after 4 retries due to , retrying +[WARNING]: Sleeping for 8 seconds, before attempting to refresh the cache again +[WARNING]: Failed to update cache after 5 retries due to , retrying +[WARNING]: Sleeping for 12 seconds, before attempting to refresh the cache again +[ERROR]: Task failed: Module failed: Failed to update apt cache after 5 retries: +Origin: /Users/newspec/Desktop/DevOps/DevOps-Core-Course/ansible/playbooks/test_rescue.yml:21:11 + +19 state: present +20 +21 - name: Try to update apt cache (will fail) + ^ column 11 + +fatal: [lab04-vm]: FAILED! => {"changed": false, "msg": "Failed to update apt cache after 5 retries: "} + +TASK [Rescue block activated!] ******************************************************************************************************************************** +ok: [lab04-vm] => { + "msg": "RESCUE BLOCK TRIGGERED! Fixing apt sources..." +} + +TASK [Restore original sources.list] ************************************************************************************************************************** +changed: [lab04-vm] + +TASK [Run apt-get update --fix-missing] *********************************************************************************************************************** +changed: [lab04-vm] + +TASK [Retry apt update] *************************************************************************************************************************************** +changed: [lab04-vm] + +TASK [Cleanup backup file] ************************************************************************************************************************************ +changed: [lab04-vm] + +TASK [Always block executed] ********************************************************************************************************************************** +ok: [lab04-vm] => { + "msg": "Always block runs regardless of success/failure" +} + +PLAY RECAP **************************************************************************************************************************************************** +lab04-vm : ok=9 changed=6 unreachable=0 failed=0 skipped=0 rescued=1 ignored=0 +``` + +### 1.5 List of all available tags (--list-tags output) +```bash +ansible-playbook playbooks/provision.yml --list-tags + +playbook: playbooks/provision.yml + + play #1 (webservers): Provision web servers TAGS: [] + TASK TAGS: [common, config, docker, docker_config, docker_install, packages, users] +``` +## 2.5 Output showing Docker Compose deployment success +```bash +PLAY [Deploy application] *************************************************************************************************************************************************** + +TASK [Gathering Facts] ****************************************************************************************************************************************************** +ok: [lab04-vm] + +TASK [docker : Install prerequisites for Docker] **************************************************************************************************************************** +ok: [lab04-vm] + +TASK [docker : Create directory for Docker GPG key] ************************************************************************************************************************* +ok: [lab04-vm] + +TASK [docker : Add Docker GPG key] ****************************************************************************************************************************************** +ok: [lab04-vm] + +TASK [docker : Get Ubuntu release codename] ********************************************************************************************************************************* +ok: [lab04-vm] + +TASK [docker : Get system architecture] ************************************************************************************************************************************* +ok: [lab04-vm] + +TASK [docker : Add Docker repository] *************************************************************************************************************************************** +ok: [lab04-vm] + +TASK [docker : Install Docker packages] ************************************************************************************************************************************* +ok: [lab04-vm] + +TASK [docker : Install python3-docker for Ansible docker modules] *********************************************************************************************************** +ok: [lab04-vm] + +TASK [docker : Ensure Docker service is enabled and started] **************************************************************************************************************** +ok: [lab04-vm] + +TASK [docker : Log Docker installation completion] ************************************************************************************************************************** +[WARNING]: Deprecation warnings can be disabled by setting `deprecation_warnings=False` in ansible.cfg. +[DEPRECATION WARNING]: INJECT_FACTS_AS_VARS default to `True` is deprecated, top-level facts will not be auto injected after the change. This feature will be removed from ansible-core version 2.24. +Origin: /Users/newspec/Desktop/DevOps/DevOps-Core-Course/ansible/roles/docker/tasks/main.yml:99:18 + +97 - name: Log Docker installation completion +98 copy: +99 content: "Docker installation completed at {{ ansible_date_time.iso8601 }}\n" + ^ column 18 + +Use `ansible_facts["fact_name"]` (no `ansible_` prefix) instead. + +changed: [lab04-vm] + +TASK [docker : Add user to docker group] ************************************************************************************************************************************ +ok: [lab04-vm] + +TASK [docker : Verify Docker installation] ********************************************************************************************************************************** +ok: [lab04-vm] + +TASK [docker : Display Docker version] ************************************************************************************************************************************** +ok: [lab04-vm] => { + "msg": "Docker version installed: Docker version 29.2.1, build a5c7197" +} + +TASK [docker : Log Docker configuration completion] ************************************************************************************************************************* +[DEPRECATION WARNING]: INJECT_FACTS_AS_VARS default to `True` is deprecated, top-level facts will not be auto injected after the change. This feature will be removed from ansible-core version 2.24. +Origin: /Users/newspec/Desktop/DevOps/DevOps-Core-Course/ansible/roles/docker/tasks/main.yml:129:18 + +127 - name: Log Docker configuration completion +128 copy: +129 content: "Docker configuration completed at {{ ansible_date_time.iso8601 }}\n" + ^ column 18 + +Use `ansible_facts["fact_name"]` (no `ansible_` prefix) instead. + +changed: [lab04-vm] + +TASK [web_app : Include wipe tasks] ***************************************************************************************************************************************** +included: /Users/newspec/Desktop/DevOps/DevOps-Core-Course/ansible/roles/web_app/tasks/wipe.yml for lab04-vm + +TASK [web_app : Stop and remove containers with Docker Compose] ************************************************************************************************************* +skipping: [lab04-vm] + +TASK [web_app : Remove docker-compose file] ********************************************************************************************************************************* +skipping: [lab04-vm] + +TASK [web_app : Remove application directory] ******************************************************************************************************************************* +skipping: [lab04-vm] + +TASK [web_app : Log wipe completion] **************************************************************************************************************************************** +skipping: [lab04-vm] + +TASK [web_app : Check if application is already running] ******************************************************************************************************************** +ok: [lab04-vm] + +TASK [web_app : Display current container status] *************************************************************************************************************************** +ok: [lab04-vm] => { + "msg": "Container devops-python is running" +} + +TASK [web_app : Create application directory] ******************************************************************************************************************************* +ok: [lab04-vm] + +TASK [web_app : Template docker-compose file] ******************************************************************************************************************************* +ok: [lab04-vm] + +TASK [web_app : Log in to Docker Hub] *************************************************************************************************************************************** +ok: [lab04-vm] + +TASK [web_app : Deploy with Docker Compose] ********************************************************************************************************************************* +ok: [lab04-vm] + +TASK [web_app : Display deployment result] ********************************************************************************************************************************** +ok: [lab04-vm] => { + "msg": "Deployment unchanged - containers are up to date" +} + +TASK [web_app : Wait for application port to be available (on target VM)] *************************************************************************************************** +ok: [lab04-vm] + +TASK [web_app : Verify health endpoint (from target VM)] ******************************************************************************************************************** +ok: [lab04-vm] + +TASK [web_app : Display health check result] ******************************************************************************************************************************** +ok: [lab04-vm] => { + "msg": "Application is healthy: {'status': 'healthy', 'timestamp': '2026-03-01T17:34:13.948071+00:00', 'uptime_seconds': 147}" +} + +PLAY RECAP ****************************************************************************************************************************************************************** +lab04-vm : ok=26 changed=2 unreachable=0 failed=0 skipped=4 rescued=0 ignored=0 +``` + +## 2.6 Idempotency proof (second run shows "ok" not "changed") +First run: +``` bash +PLAY [Deploy application] *************************************************************************************************************************************************** + +TASK [Gathering Facts] ****************************************************************************************************************************************************** +ok: [lab04-vm] + +TASK [docker : Install prerequisites for Docker] **************************************************************************************************************************** +ok: [lab04-vm] + +TASK [docker : Create directory for Docker GPG key] ************************************************************************************************************************* +ok: [lab04-vm] + +TASK [docker : Add Docker GPG key] ****************************************************************************************************************************************** +ok: [lab04-vm] + +TASK [docker : Get Ubuntu release codename] ********************************************************************************************************************************* +ok: [lab04-vm] + +TASK [docker : Get system architecture] ************************************************************************************************************************************* +ok: [lab04-vm] + +TASK [docker : Add Docker repository] *************************************************************************************************************************************** +changed: [lab04-vm] + +TASK [docker : Install Docker packages] ************************************************************************************************************************************* +changed: [lab04-vm] + +TASK [docker : Install python3-docker for Ansible docker modules] *********************************************************************************************************** +ok: [lab04-vm] + +TASK [docker : Ensure Docker service is enabled and started] **************************************************************************************************************** +ok: [lab04-vm] + +TASK [docker : Log Docker installation completion] ************************************************************************************************************************** +[WARNING]: Deprecation warnings can be disabled by setting `deprecation_warnings=False` in ansible.cfg. +[DEPRECATION WARNING]: INJECT_FACTS_AS_VARS default to `True` is deprecated, top-level facts will not be auto injected after the change. This feature will be removed from ansible-core version 2.24. +Origin: /Users/newspec/Desktop/DevOps/DevOps-Core-Course/ansible/roles/docker/tasks/main.yml:99:18 + +97 - name: Log Docker installation completion +98 copy: +99 content: "Docker installation completed at {{ ansible_date_time.iso8601 }}\n" + ^ column 18 + +Use `ansible_facts["fact_name"]` (no `ansible_` prefix) instead. + +changed: [lab04-vm] + +TASK [docker : Add user to docker group] ************************************************************************************************************************************ +changed: [lab04-vm] + +TASK [docker : Verify Docker installation] ********************************************************************************************************************************** +ok: [lab04-vm] + +TASK [docker : Display Docker version] ************************************************************************************************************************************** +ok: [lab04-vm] => { + "msg": "Docker version installed: Docker version 29.2.1, build a5c7197" +} + +TASK [docker : Log Docker configuration completion] ************************************************************************************************************************* +[DEPRECATION WARNING]: INJECT_FACTS_AS_VARS default to `True` is deprecated, top-level facts will not be auto injected after the change. This feature will be removed from ansible-core version 2.24. +Origin: /Users/newspec/Desktop/DevOps/DevOps-Core-Course/ansible/roles/docker/tasks/main.yml:129:18 + +127 - name: Log Docker configuration completion +128 copy: +129 content: "Docker configuration completed at {{ ansible_date_time.iso8601 }}\n" + ^ column 18 + +Use `ansible_facts["fact_name"]` (no `ansible_` prefix) instead. + +changed: [lab04-vm] + +TASK [web_app : Include wipe tasks] ***************************************************************************************************************************************** +included: /Users/newspec/Desktop/DevOps/DevOps-Core-Course/ansible/roles/web_app/tasks/wipe.yml for lab04-vm + +TASK [web_app : Check if docker-compose file exists] ************************************************************************************************************************ +skipping: [lab04-vm] + +TASK [web_app : Stop and remove containers with Docker Compose] ************************************************************************************************************* +skipping: [lab04-vm] + +TASK [web_app : Stop container manually if compose file doesn't exist] ****************************************************************************************************** +skipping: [lab04-vm] + +TASK [web_app : Remove docker-compose file] ********************************************************************************************************************************* +skipping: [lab04-vm] + +TASK [web_app : Remove application directory] ******************************************************************************************************************************* +skipping: [lab04-vm] + +TASK [web_app : Log wipe completion] **************************************************************************************************************************************** +skipping: [lab04-vm] + +TASK [web_app : Check if application is already running] ******************************************************************************************************************** +ok: [lab04-vm] + +TASK [web_app : Display current container status] *************************************************************************************************************************** +ok: [lab04-vm] => { + "msg": "Container devops-python is not running" +} + +TASK [web_app : Create application directory] ******************************************************************************************************************************* +changed: [lab04-vm] + +TASK [web_app : Template docker-compose file] ******************************************************************************************************************************* +changed: [lab04-vm] + +TASK [web_app : Log in to Docker Hub] *************************************************************************************************************************************** +ok: [lab04-vm] + +TASK [web_app : Deploy with Docker Compose] ********************************************************************************************************************************* +changed: [lab04-vm] + +TASK [web_app : Display deployment result] ********************************************************************************************************************************** +ok: [lab04-vm] => { + "msg": "Deployment changed - containers are updated" +} + +TASK [web_app : Wait for application port to be available (on target VM)] *************************************************************************************************** +ok: [lab04-vm] + +TASK [web_app : Verify health endpoint (from target VM)] ******************************************************************************************************************** +ok: [lab04-vm] + +TASK [web_app : Display health check result] ******************************************************************************************************************************** +ok: [lab04-vm] => { + "msg": "Application is healthy: {'status': 'healthy', 'timestamp': '2026-03-01T17:46:21.068443+00:00', 'uptime_seconds': 6}" +} + +RUNNING HANDLER [docker : restart docker] *********************************************************************************************************************************** +changed: [lab04-vm] + +PLAY RECAP ****************************************************************************************************************************************************************** +lab04-vm : ok=27 changed=9 unreachable=0 failed=0 skipped=6 rescued=0 ignored=0 +``` +Second run: +```bash +PLAY [Deploy application] *************************************************************************************************************************************************** + +TASK [Gathering Facts] ****************************************************************************************************************************************************** +ok: [lab04-vm] + +TASK [docker : Install prerequisites for Docker] **************************************************************************************************************************** +ok: [lab04-vm] + +TASK [docker : Create directory for Docker GPG key] ************************************************************************************************************************* +ok: [lab04-vm] + +TASK [docker : Add Docker GPG key] ****************************************************************************************************************************************** +ok: [lab04-vm] + +TASK [docker : Get Ubuntu release codename] ********************************************************************************************************************************* +ok: [lab04-vm] + +TASK [docker : Get system architecture] ************************************************************************************************************************************* +ok: [lab04-vm] + +TASK [docker : Add Docker repository] *************************************************************************************************************************************** +ok: [lab04-vm] + +TASK [docker : Install Docker packages] ************************************************************************************************************************************* +ok: [lab04-vm] + +TASK [docker : Install python3-docker for Ansible docker modules] *********************************************************************************************************** +ok: [lab04-vm] + +TASK [docker : Ensure Docker service is enabled and started] **************************************************************************************************************** +ok: [lab04-vm] + +TASK [docker : Log Docker installation completion] ************************************************************************************************************************** +[WARNING]: Deprecation warnings can be disabled by setting `deprecation_warnings=False` in ansible.cfg. +[DEPRECATION WARNING]: INJECT_FACTS_AS_VARS default to `True` is deprecated, top-level facts will not be auto injected after the change. This feature will be removed from ansible-core version 2.24. +Origin: /Users/newspec/Desktop/DevOps/DevOps-Core-Course/ansible/roles/docker/tasks/main.yml:99:18 + +97 - name: Log Docker installation completion +98 copy: +99 content: "Docker installation completed at {{ ansible_date_time.iso8601 }}\n" + ^ column 18 + +Use `ansible_facts["fact_name"]` (no `ansible_` prefix) instead. + +changed: [lab04-vm] + +TASK [docker : Add user to docker group] ************************************************************************************************************************************ +ok: [lab04-vm] + +TASK [docker : Verify Docker installation] ********************************************************************************************************************************** +ok: [lab04-vm] + +TASK [docker : Display Docker version] ************************************************************************************************************************************** +ok: [lab04-vm] => { + "msg": "Docker version installed: Docker version 29.2.1, build a5c7197" +} + +TASK [docker : Log Docker configuration completion] ************************************************************************************************************************* +[DEPRECATION WARNING]: INJECT_FACTS_AS_VARS default to `True` is deprecated, top-level facts will not be auto injected after the change. This feature will be removed from ansible-core version 2.24. +Origin: /Users/newspec/Desktop/DevOps/DevOps-Core-Course/ansible/roles/docker/tasks/main.yml:129:18 + +127 - name: Log Docker configuration completion +128 copy: +129 content: "Docker configuration completed at {{ ansible_date_time.iso8601 }}\n" + ^ column 18 + +Use `ansible_facts["fact_name"]` (no `ansible_` prefix) instead. + +changed: [lab04-vm] + +TASK [web_app : Include wipe tasks] ***************************************************************************************************************************************** +included: /Users/newspec/Desktop/DevOps/DevOps-Core-Course/ansible/roles/web_app/tasks/wipe.yml for lab04-vm + +TASK [web_app : Check if docker-compose file exists] ************************************************************************************************************************ +skipping: [lab04-vm] + +TASK [web_app : Stop and remove containers with Docker Compose] ************************************************************************************************************* +skipping: [lab04-vm] + +TASK [web_app : Stop container manually if compose file doesn't exist] ****************************************************************************************************** +skipping: [lab04-vm] + +TASK [web_app : Remove docker-compose file] ********************************************************************************************************************************* +skipping: [lab04-vm] + +TASK [web_app : Remove application directory] ******************************************************************************************************************************* +skipping: [lab04-vm] + +TASK [web_app : Log wipe completion] **************************************************************************************************************************************** +skipping: [lab04-vm] + +TASK [web_app : Check if application is already running] ******************************************************************************************************************** +ok: [lab04-vm] + +TASK [web_app : Display current container status] *************************************************************************************************************************** +ok: [lab04-vm] => { + "msg": "Container devops-python is running" +} + +TASK [web_app : Create application directory] ******************************************************************************************************************************* +ok: [lab04-vm] + +TASK [web_app : Template docker-compose file] ******************************************************************************************************************************* +ok: [lab04-vm] + +TASK [web_app : Log in to Docker Hub] *************************************************************************************************************************************** +ok: [lab04-vm] + +TASK [web_app : Deploy with Docker Compose] ********************************************************************************************************************************* +ok: [lab04-vm] + +TASK [web_app : Display deployment result] ********************************************************************************************************************************** +ok: [lab04-vm] => { + "msg": "Deployment unchanged - containers are up to date" +} + +TASK [web_app : Wait for application port to be available (on target VM)] *************************************************************************************************** +ok: [lab04-vm] + +TASK [web_app : Verify health endpoint (from target VM)] ******************************************************************************************************************** +ok: [lab04-vm] + +TASK [web_app : Display health check result] ******************************************************************************************************************************** +ok: [lab04-vm] => { + "msg": "Application is healthy: {'status': 'healthy', 'timestamp': '2026-03-01T17:48:42.025713+00:00', 'uptime_seconds': 135}" +} + +PLAY RECAP ****************************************************************************************************************************************************************** +lab04-vm : ok=26 changed=2 unreachable=0 failed=0 skipped=6 rescued=0 ignored=0 +``` + +## 2.7 Application running and accessible +```bash +ubuntu@lab04-vm:~$ docker ps +CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES +1253f65bdeb0 newspec/python_app:latest "python app.py" 4 minutes ago Up 4 minutes 0.0.0.0:8000->8000/tcp, [::]:8000->8000/tcp devops-python +ubuntu@lab04-vm:~$ docker compose -f /opt/devops-python/docker-compose.yml ps +NAME IMAGE COMMAND SERVICE CREATED STATUS PORTS +devops-python newspec/python_app:latest "python app.py" devops-python 5 minutes ago Up 4 minutes 0.0.0.0:8000->8000/tcp, [::]:8000->8000/tcp +ubuntu@lab04-vm:~$ curl http://localhost:8000 +{"service":{"name":"devops-info-service","version":"1.0.0","description":"DevOps course info service","framework":"FastAPI"},"system":{"hostname":"1253f65bdeb0","platform":"Linux","platform_version":"#100-Ubuntu SMP PREEMPT_DYNAMIC Tue Jan 13 16:40:06 UTC 2026","architecture":"x86_64","cpu_count":2,"python_version":"3.12.12"},"runtime":{"uptime_seconds":290,"uptime_human":"0 hours, 4 minutes","current_time":"2026-03-01T17:51:16.359911+00:00","timezone":"UTC"},"request":{"client_ip":"172.19.0.1","user_agent":"curl/8.5.0","method":"GET","path":"/"},"endpoints":[{"path":"/","method":"GET","description":"Service information"},{"path":"/health","method":"GET","description":"Health check"} +``` +### 3.4 Test Results + +**Scenario 1: Normal deployment (wipe should NOT run)** +```bash +ansible-playbook playbooks/deploy.yml +PLAY [Deploy application] *************************************************************************************************************************************************** + +TASK [Gathering Facts] ****************************************************************************************************************************************************** +ok: [lab04-vm] + +TASK [docker : Install prerequisites for Docker] **************************************************************************************************************************** +ok: [lab04-vm] + +TASK [docker : Create directory for Docker GPG key] ************************************************************************************************************************* +ok: [lab04-vm] + +TASK [docker : Add Docker GPG key] ****************************************************************************************************************************************** +ok: [lab04-vm] + +TASK [docker : Get Ubuntu release codename] ********************************************************************************************************************************* +ok: [lab04-vm] + +TASK [docker : Get system architecture] ************************************************************************************************************************************* +ok: [lab04-vm] + +TASK [docker : Add Docker repository] *************************************************************************************************************************************** +ok: [lab04-vm] + +TASK [docker : Install Docker packages] ************************************************************************************************************************************* +ok: [lab04-vm] + +TASK [docker : Install python3-docker for Ansible docker modules] *********************************************************************************************************** +ok: [lab04-vm] + +TASK [docker : Ensure Docker service is enabled and started] **************************************************************************************************************** +ok: [lab04-vm] + +TASK [docker : Log Docker installation completion] ************************************************************************************************************************** +[WARNING]: Deprecation warnings can be disabled by setting `deprecation_warnings=False` in ansible.cfg. +[DEPRECATION WARNING]: INJECT_FACTS_AS_VARS default to `True` is deprecated, top-level facts will not be auto injected after the change. This feature will be removed from ansible-core version 2.24. +Origin: /Users/newspec/Desktop/DevOps/DevOps-Core-Course/ansible/roles/docker/tasks/main.yml:99:18 + +97 - name: Log Docker installation completion +98 copy: +99 content: "Docker installation completed at {{ ansible_date_time.iso8601 }}\n" + ^ column 18 + +Use `ansible_facts["fact_name"]` (no `ansible_` prefix) instead. + +changed: [lab04-vm] + +TASK [docker : Add user to docker group] ************************************************************************************************************************************ +ok: [lab04-vm] + +TASK [docker : Verify Docker installation] ********************************************************************************************************************************** +ok: [lab04-vm] + +TASK [docker : Display Docker version] ************************************************************************************************************************************** +ok: [lab04-vm] => { + "msg": "Docker version installed: Docker version 29.2.1, build a5c7197" +} + +TASK [docker : Log Docker configuration completion] ************************************************************************************************************************* +[DEPRECATION WARNING]: INJECT_FACTS_AS_VARS default to `True` is deprecated, top-level facts will not be auto injected after the change. This feature will be removed from ansible-core version 2.24. +Origin: /Users/newspec/Desktop/DevOps/DevOps-Core-Course/ansible/roles/docker/tasks/main.yml:129:18 + +127 - name: Log Docker configuration completion +128 copy: +129 content: "Docker configuration completed at {{ ansible_date_time.iso8601 }}\n" + ^ column 18 + +Use `ansible_facts["fact_name"]` (no `ansible_` prefix) instead. + +changed: [lab04-vm] + +TASK [web_app : Include wipe tasks] ***************************************************************************************************************************************** +included: /Users/newspec/Desktop/DevOps/DevOps-Core-Course/ansible/roles/web_app/tasks/wipe.yml for lab04-vm + +TASK [web_app : Check if docker-compose file exists] ************************************************************************************************************************ +skipping: [lab04-vm] + +TASK [web_app : Stop and remove containers with Docker Compose] ************************************************************************************************************* +skipping: [lab04-vm] + +TASK [web_app : Stop container manually if compose file doesn't exist] ****************************************************************************************************** +skipping: [lab04-vm] + +TASK [web_app : Remove docker-compose file] ********************************************************************************************************************************* +skipping: [lab04-vm] + +TASK [web_app : Remove application directory] ******************************************************************************************************************************* +skipping: [lab04-vm] + +TASK [web_app : Log wipe completion] **************************************************************************************************************************************** +skipping: [lab04-vm] + +TASK [web_app : Check if application is already running] ******************************************************************************************************************** +ok: [lab04-vm] + +TASK [web_app : Display current container status] *************************************************************************************************************************** +ok: [lab04-vm] => { + "msg": "Container devops-python is running" +} + +TASK [web_app : Create application directory] ******************************************************************************************************************************* +ok: [lab04-vm] + +TASK [web_app : Template docker-compose file] ******************************************************************************************************************************* +ok: [lab04-vm] + +TASK [web_app : Log in to Docker Hub] *************************************************************************************************************************************** +ok: [lab04-vm] + +TASK [web_app : Deploy with Docker Compose] ********************************************************************************************************************************* +ok: [lab04-vm] + +TASK [web_app : Display deployment result] ********************************************************************************************************************************** +ok: [lab04-vm] => { + "msg": "Deployment unchanged - containers are up to date" +} + +TASK [web_app : Wait for application port to be available (on target VM)] *************************************************************************************************** +ok: [lab04-vm] + +TASK [web_app : Verify health endpoint (from target VM)] ******************************************************************************************************************** +ok: [lab04-vm] + +TASK [web_app : Display health check result] ******************************************************************************************************************************** +ok: [lab04-vm] => { + "msg": "Application is healthy: {'status': 'healthy', 'timestamp': '2026-03-01T18:04:45.889812+00:00', 'uptime_seconds': 1099}" +} + +PLAY RECAP ****************************************************************************************************************************************************************** +lab04-vm : ok=26 changed=2 unreachable=0 failed=0 skipped=6 rescued=0 ignored=0 + +``` +```bash +ssh ubuntu@93.77.180.155 "docker ps" +CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES +1253f65bdeb0 newspec/python_app:latest "python app.py" 19 minutes ago Up 19 minutes 0.0.0.0:8000->8000/tcp, [::]:8000->8000/tcp devops-python +``` + Result: App deploys normally, wipe tasks skipped (tag not specified) + +**Scenario 2: Wipe only (remove existing deployment)** +```bash +ansible-playbook playbooks/deploy.yml \ + -e "web_app_wipe=true" \ + --tags web_app_wipe + PLAY [Deploy application] *************************************************************************************************************************************************** + +TASK [Gathering Facts] ****************************************************************************************************************************************************** +ok: [lab04-vm] + +TASK [web_app : Include wipe tasks] ***************************************************************************************************************************************** +included: /Users/newspec/Desktop/DevOps/DevOps-Core-Course/ansible/roles/web_app/tasks/wipe.yml for lab04-vm + +TASK [web_app : Check if docker-compose file exists] ************************************************************************************************************************ +ok: [lab04-vm] + +TASK [web_app : Stop and remove containers with Docker Compose] ************************************************************************************************************* +changed: [lab04-vm] + +TASK [web_app : Stop container manually if compose file doesn't exist] ****************************************************************************************************** +skipping: [lab04-vm] + +TASK [web_app : Remove docker-compose file] ********************************************************************************************************************************* +changed: [lab04-vm] + +TASK [web_app : Remove application directory] ******************************************************************************************************************************* +changed: [lab04-vm] + +TASK [web_app : Log wipe completion] **************************************************************************************************************************************** +ok: [lab04-vm] => { + "msg": "Application devops-python wiped successfully from /opt/devops-python" +} + +PLAY RECAP ****************************************************************************************************************************************************************** +lab04-vm : ok=7 changed=3 unreachable=0 failed=0 skipped=1 rescued=0 ignored=0 +``` +```bash +ssh ubuntu@93.77.180.155 "docker ps" +CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES +``` +```bash +ssh ubuntu@93.77.180.155 "ls /opt" +containerd +``` +Result: App removed, deployment skipped + +**Scenario 3: Clean reinstallation (wipe β†’ deploy)** +```bash +ansible-playbook playbooks/deploy.yml \ + -e "web_app_wipe=true" +PLAY [Deploy application] *************************************************************************************************************************************************** + +TASK [Gathering Facts] ****************************************************************************************************************************************************** +ok: [lab04-vm] + +TASK [docker : Install prerequisites for Docker] **************************************************************************************************************************** +ok: [lab04-vm] + +TASK [docker : Create directory for Docker GPG key] ************************************************************************************************************************* +ok: [lab04-vm] + +TASK [docker : Add Docker GPG key] ****************************************************************************************************************************************** +ok: [lab04-vm] + +TASK [docker : Get Ubuntu release codename] ********************************************************************************************************************************* +ok: [lab04-vm] + +TASK [docker : Get system architecture] ************************************************************************************************************************************* +ok: [lab04-vm] + +TASK [docker : Add Docker repository] *************************************************************************************************************************************** +ok: [lab04-vm] + +TASK [docker : Install Docker packages] ************************************************************************************************************************************* +ok: [lab04-vm] + +TASK [docker : Install python3-docker for Ansible docker modules] *********************************************************************************************************** +ok: [lab04-vm] + +TASK [docker : Ensure Docker service is enabled and started] **************************************************************************************************************** +ok: [lab04-vm] + +TASK [docker : Log Docker installation completion] ************************************************************************************************************************** +[WARNING]: Deprecation warnings can be disabled by setting `deprecation_warnings=False` in ansible.cfg. +[DEPRECATION WARNING]: INJECT_FACTS_AS_VARS default to `True` is deprecated, top-level facts will not be auto injected after the change. This feature will be removed from ansible-core version 2.24. +Origin: /Users/newspec/Desktop/DevOps/DevOps-Core-Course/ansible/roles/docker/tasks/main.yml:99:18 + +97 - name: Log Docker installation completion +98 copy: +99 content: "Docker installation completed at {{ ansible_date_time.iso8601 }}\n" + ^ column 18 + +Use `ansible_facts["fact_name"]` (no `ansible_` prefix) instead. + +changed: [lab04-vm] + +TASK [docker : Add user to docker group] ************************************************************************************************************************************ +ok: [lab04-vm] + +TASK [docker : Verify Docker installation] ********************************************************************************************************************************** +ok: [lab04-vm] + +TASK [docker : Display Docker version] ************************************************************************************************************************************** +ok: [lab04-vm] => { + "msg": "Docker version installed: Docker version 29.2.1, build a5c7197" +} + +TASK [docker : Log Docker configuration completion] ************************************************************************************************************************* +[DEPRECATION WARNING]: INJECT_FACTS_AS_VARS default to `True` is deprecated, top-level facts will not be auto injected after the change. This feature will be removed from ansible-core version 2.24. +Origin: /Users/newspec/Desktop/DevOps/DevOps-Core-Course/ansible/roles/docker/tasks/main.yml:129:18 + +127 - name: Log Docker configuration completion +128 copy: +129 content: "Docker configuration completed at {{ ansible_date_time.iso8601 }}\n" + ^ column 18 + +Use `ansible_facts["fact_name"]` (no `ansible_` prefix) instead. + +changed: [lab04-vm] + +TASK [web_app : Include wipe tasks] ***************************************************************************************************************************************** +included: /Users/newspec/Desktop/DevOps/DevOps-Core-Course/ansible/roles/web_app/tasks/wipe.yml for lab04-vm + +TASK [web_app : Check if docker-compose file exists] ************************************************************************************************************************ +ok: [lab04-vm] + +TASK [web_app : Stop and remove containers with Docker Compose] ************************************************************************************************************* +skipping: [lab04-vm] + +TASK [web_app : Stop container manually if compose file doesn't exist] ****************************************************************************************************** +ok: [lab04-vm] + +TASK [web_app : Remove docker-compose file] ********************************************************************************************************************************* +ok: [lab04-vm] + +TASK [web_app : Remove application directory] ******************************************************************************************************************************* +ok: [lab04-vm] + +TASK [web_app : Log wipe completion] **************************************************************************************************************************************** +ok: [lab04-vm] => { + "msg": "Application devops-python wiped successfully from /opt/devops-python" +} + +TASK [web_app : Check if application is already running] ******************************************************************************************************************** +ok: [lab04-vm] + +TASK [web_app : Display current container status] *************************************************************************************************************************** +ok: [lab04-vm] => { + "msg": "Container devops-python is not running" +} + +TASK [web_app : Create application directory] ******************************************************************************************************************************* +changed: [lab04-vm] + +TASK [web_app : Template docker-compose file] ******************************************************************************************************************************* +changed: [lab04-vm] + +TASK [web_app : Log in to Docker Hub] *************************************************************************************************************************************** +ok: [lab04-vm] + +TASK [web_app : Deploy with Docker Compose] ********************************************************************************************************************************* +changed: [lab04-vm] + +TASK [web_app : Display deployment result] ********************************************************************************************************************************** +ok: [lab04-vm] => { + "msg": "Deployment changed - containers are updated" +} + +TASK [web_app : Wait for application port to be available (on target VM)] *************************************************************************************************** +ok: [lab04-vm] + +TASK [web_app : Verify health endpoint (from target VM)] ******************************************************************************************************************** +ok: [lab04-vm] + +TASK [web_app : Display health check result] ******************************************************************************************************************************** +ok: [lab04-vm] => { + "msg": "Application is healthy: {'status': 'healthy', 'timestamp': '2026-03-01T18:20:21.216078+00:00', 'uptime_seconds': 6}" +} + +PLAY RECAP ****************************************************************************************************************************************************************** +lab04-vm : ok=31 changed=5 unreachable=0 failed=0 skipped=1 rescued=0 ignored=0 +``` +```bash +ssh ubuntu@93.77.180.155 "docker ps" +CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES +5f5c4479353f newspec/python_app:latest "python app.py" 29 seconds ago Up 28 seconds 0.0.0.0:8000->8000/tcp, [::]:8000->8000/tcp devops-python +``` +Result: Old app removed, new app deployed (clean reinstall) + +**Scenario 4a: Safety check - Tag specified but variable false** +```bash +ansible-playbook playbooks/deploy.yml --tags web_app_wipe +PLAY [Deploy application] *************************************************************************************************************************************************** + +TASK [Gathering Facts] ****************************************************************************************************************************************************** +ok: [lab04-vm] + +TASK [web_app : Include wipe tasks] ***************************************************************************************************************************************** +included: /Users/newspec/Desktop/DevOps/DevOps-Core-Course/ansible/roles/web_app/tasks/wipe.yml for lab04-vm + +TASK [web_app : Check if docker-compose file exists] ************************************************************************************************************************ +skipping: [lab04-vm] + +TASK [web_app : Stop and remove containers with Docker Compose] ************************************************************************************************************* +skipping: [lab04-vm] + +TASK [web_app : Stop container manually if compose file doesn't exist] ****************************************************************************************************** +skipping: [lab04-vm] + +TASK [web_app : Remove docker-compose file] ********************************************************************************************************************************* +skipping: [lab04-vm] + +TASK [web_app : Remove application directory] ******************************************************************************************************************************* +skipping: [lab04-vm] + +TASK [web_app : Log wipe completion] **************************************************************************************************************************************** +skipping: [lab04-vm] + +PLAY RECAP ****************************************************************************************************************************************************************** +lab04-vm : ok=2 changed=0 unreachable=0 failed=0 skipped=6 rescued=0 ignored=0 +``` +Result: Wipe tasks skipped (when condition blocks it), deployment runs normally + +**Scenario 4b: Safety check - Variable true, deployment skipped** +```bash +ansible-playbook playbooks/deploy.yml \ + -e "web_app_wipe=true" \ + --tags web_app_wipe +PLAY [Deploy application] *************************************************************************************************************************************************** + +TASK [Gathering Facts] ****************************************************************************************************************************************************** +ok: [lab04-vm] + +TASK [web_app : Include wipe tasks] ***************************************************************************************************************************************** +included: /Users/newspec/Desktop/DevOps/DevOps-Core-Course/ansible/roles/web_app/tasks/wipe.yml for lab04-vm + +TASK [web_app : Check if docker-compose file exists] ************************************************************************************************************************ +ok: [lab04-vm] + +TASK [web_app : Stop and remove containers with Docker Compose] ************************************************************************************************************* +changed: [lab04-vm] + +TASK [web_app : Stop container manually if compose file doesn't exist] ****************************************************************************************************** +skipping: [lab04-vm] + +TASK [web_app : Remove docker-compose file] ********************************************************************************************************************************* +changed: [lab04-vm] + +TASK [web_app : Remove application directory] ******************************************************************************************************************************* +changed: [lab04-vm] + +TASK [web_app : Log wipe completion] **************************************************************************************************************************************** +ok: [lab04-vm] => { + "msg": "Application devops-python wiped successfully from /opt/devops-python" +} + +PLAY RECAP ****************************************************************************************************************************************************************** +lab04-vm : ok=7 changed=3 unreachable=0 failed=0 skipped=1 rescued=0 ignored=0 +``` +Result: Only wipe runs, no deployment + +#### Screenshot of application running after clean reinstall +![alt text](image.png) +### 4.3 Evidence of Automated Deployments + +#### Screenshot of successful workflow run +![alt text](image-1.png) + +#### Output logs showing ansible-lint passing +``` +Run cd ansible +Passed: 0 failure(s), 0 warning(s) in 7 files processed of 7 encountered. Last profile that met the validation criteria was 'production'. +``` + +#### Output logs showing ansible-playbook execution +``` +Run cd ansible + +PLAY [Deploy Python Application] *********************************************** + +TASK [Gathering Facts] ********************************************************* +ok: [lab04-vm] + +TASK [web_app : Check if application is already running] *********************** +ok: [lab04-vm] + +TASK [web_app : Display current container status] ****************************** +ok: [lab04-vm] => { + "msg": "Container devops-python is running" +} + +TASK [web_app : Create application directory] ********************************** +ok: [lab04-vm] + +TASK [web_app : Template docker-compose file] ********************************** +changed: [lab04-vm] + +TASK [web_app : Log in to Docker Hub] ****************************************** +ok: [lab04-vm] + +TASK [web_app : Deploy with Docker Compose] ************************************ +ok: [lab04-vm] + +TASK [web_app : Display deployment result] ************************************* +ok: [lab04-vm] => { + "msg": "Deployment unchanged - containers are up to date" +} + +TASK [web_app : Wait for application port to be available (on target VM)] ****** +ok: [lab04-vm] + +TASK [web_app : Verify health endpoint (from target VM)] *********************** +ok: [lab04-vm] + +TASK [web_app : Display health check result] *********************************** +ok: [lab04-vm] => { + "msg": "Application is healthy: {'status': 'healthy', 'timestamp': '2026-03-01T19:30:19.234174+00:00', 'uptime_seconds': 3801}" +} + +PLAY RECAP ********************************************************************* +lab04-vm : ok=11 changed=1 unreachable=0 failed=0 skipped=0 rescued=0 ignored=0 +``` +#### Verification step output showing app responding +``` +Run if [ -n "***" ]; then + % Total % Received % Xferd Average Speed Time Time Time Current + Dload Upload Total Spent Left Speed + + 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 +100 699 100 699 0 0 1891 0 --:--:-- --:--:-- --:--:-- 1889 +100 699 100 699 0 0 1891 0 --:--:-- --:--:-- --:--:-- 1889 + % Total % Received % Xferd Average Speed Time Time Time Current + Dload Upload Total Spent Left Speed + + 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 +100 89 100 89 0 0 235 0 --:--:-- --:--:-- --:--:-- 236 +{"service":{"name":"devops-info-service","version":"1.0.0","description":"DevOps course info service","framework":"FastAPI"},"system":{"hostname":"01f6effa0e13","platform":"Linux","platform_version":"#100-Ubuntu SMP PREEMPT_DYNAMIC Tue Jan 13 16:40:06 UTC 2026","architecture":"x86_64","cpu_count":2,"python_version":"3.12.12"},"runtime":{"uptime_seconds":3812,"uptime_human":"1 hours, 3 minutes","current_time":"2026-03-01T19:30:30.146722+00:00","timezone":"UTC"},"re***uest":{"client_ip":"52.159.247.196","user_agent":"curl/8.5.0","method":"GET","path":"/"},"endpoints":[{"path":"/","method":"GET","description":"Service information"},{"path":"/health","method":"GET","description":"Health check"}]}{"status":"healthy","timestamp":"2026-03-01T19:30:30.531667+00:00","uptime_seconds":3812} +``` +#### Status badge in README showing passing +[![Ansible Deployment - Python App](https://github.com/newspec/DevOps-Core-Course/actions/workflows/ansible-deploy.yml/badge.svg)](https://github.com/newspec/DevOps-Core-Course/actions/workflows/ansible-deploy.yml) + +# Challenges & Solutions + +## Challenge 1: ansible-lint Compliance (51 violations) + +**Problem:** +After implementing all lab tasks, running `ansible-lint` revealed 51 violations across multiple categories: +- FQCN (Fully Qualified Collection Names) - 30+ violations +- Variable naming without role prefix - 15+ violations +- YAML truthy values (`yes/no` instead of `true/false`) - 3 violations +- Task naming conventions - 1 violation +- Missing newlines at end of files - 7 violations +- Improper use of `ignore_errors` - 2 violations + +**Solution:** +Systematically fixed all violations in multiple passes: + +1. **FQCN Compliance:** + - Added `ansible.builtin.` prefix to all builtin modules + - Changed `file` β†’ `ansible.builtin.file` + - Changed `command` β†’ `ansible.builtin.command` + - Changed `debug` β†’ `ansible.builtin.debug` + - Changed `template` β†’ `ansible.builtin.template` + - Changed `stat` β†’ `ansible.builtin.stat` + +2. **Variable Naming Convention:** + - Renamed all role variables with `web_app_` prefix + - `app_name` β†’ `web_app_name` + - `docker_image` β†’ `web_app_docker_image` + - `compose_project_dir` β†’ `web_app_compose_project_dir` + - Updated all references in templates, tasks, and variable files + +3. **YAML Best Practices:** + - Changed `yes/no` to `true/false` throughout + - Added newlines at end of all YAML files + - Fixed task naming: `restart application` β†’ `Restart application` + +4. **Error Handling:** + - Replaced `ignore_errors: yes` with `failed_when: false` + - More explicit about when failures are acceptable + +5. **Block Key Ordering:** + - Reordered block keys: `when`, `become`, `tags` before `block:` + - Ensures proper YAML structure + +**Result:** All 51 violations resolved. ansible-lint now passes with 0 errors, 0 warnings. + +--- + +## Challenge 2: GitHub Actions Workflow Validation Errors + +**Problem:** +Initial workflow file had validation errors: +```yaml +- name: Setup SSH + if: ${{ secrets.SSH_PRIVATE_KEY != '' }} # ❌ Invalid + run: | + echo "${{ secrets.SSH_PRIVATE_KEY }}" > ~/.ssh/id_rsa +``` + +Error: `secrets` context cannot be used in `if` conditions at step level. + +**Solution:** +Moved conditionals inside bash scripts: +```yaml +- name: Setup SSH + run: | + if [ -n "${{ secrets.SSH_PRIVATE_KEY }}" ]; then + echo "${{ secrets.SSH_PRIVATE_KEY }}" > ~/.ssh/id_rsa + chmod 600 ~/.ssh/id_rsa + fi +``` + +**Lesson Learned:** GitHub Actions has strict context usage rules. Use bash conditionals for secret checks. + +--- + + +## Challenge 3: Undefined Variable in Workflow + +**Problem:** +Deployment failed with error: +``` +'dockerhub_username' is undefined +``` + +The variable was defined in `group_vars/all.yml` but not loaded in GitHub Actions workflow. + +**Solution:** +Added `-e @group_vars/all.yml` to ansible-playbook command: +```yaml +- name: Deploy Python App with Ansible + run: | + cd ansible + ansible-playbook playbooks/deploy_python.yml \ + --vault-password-file /tmp/vault_pass \ + --tags "app_deploy" \ + -e @group_vars/all.yml # Load variables +``` + +# Research Answers +**Q: What happens if rescue block also fails?** +A: If the rescue block fails, the entire block fails and Ansible will stop execution (unless `ignore_errors: yes` is set). The always block will still execute before failure. This is why rescue blocks should be simple and reliable. + +**Q: Can you have nested blocks?** +A: Yes, blocks can be nested. However, it's generally not recommended as it makes playbooks harder to read. Better to use separate blocks or include_tasks for complex logic. + +**Q: How do tags inherit to tasks within blocks?** +A: Tags applied to a block are automatically inherited by all tasks within that block. This is more efficient than tagging each task individually. Child tasks can have additional tags beyond the block's tags. + +**Q: What's the difference between `restart: always` and `restart: unless-stopped`?** +A: +- `always`: Container restarts even if manually stopped, including after system reboot +- `unless-stopped`: Container restarts automatically EXCEPT when manually stopped. After reboot, it won't start if it was manually stopped before +- `unless-stopped` is better for production as it respects manual intervention + +**Q: How do Docker Compose networks differ from Docker bridge networks?** +A: +- Docker Compose creates isolated networks per project by default +- Compose networks have automatic DNS resolution between services +- Bridge networks are more manual and require explicit linking +- Compose networks are automatically cleaned up when project is removed + +**Q: Can you reference Ansible Vault variables in the template?** +A: Yes! Vault variables are decrypted before template rendering, so they can be used like any other variable in Jinja2 templates. Example: `{{ vault_secret_key }}` + +**1. Why use both variable AND tag?** +A: Double safety mechanism prevents accidental data loss: +- Variable alone: Could be set accidentally in vars file +- Tag alone: Could be run without realizing consequences +- Both together: Requires explicit, conscious decision + +**2. What's the difference between `never` tag and this approach?** +A: +- `never` tag: Tasks NEVER run unless explicitly called with `--tags never` +- Our approach: Tasks can run in two scenarios (wipe-only OR clean-install) +- Our approach is more flexible for the clean reinstall use case + +**3. Why must wipe logic come BEFORE deployment in main.yml?** +A: To support the clean reinstall scenario where we want to: +1. First wipe the old installation +2. Then deploy the new installation +If wipe came after, we'd deploy then immediately wipe! + +**4. When would you want clean reinstallation vs. rolling update?** +A: +- Clean reinstall: Major version changes, corrupted state, testing from scratch +- Rolling update: Minor updates, zero-downtime requirements, production environments + +**5. How would you extend this to wipe Docker images and volumes too?** +A: Add tasks to wipe.yml: +```yaml +- name: Remove Docker images + community.docker.docker_image: + name: "{{ docker_image }}" + tag: "{{ docker_tag }}" + state: absent + +- name: Remove Docker volumes + community.docker.docker_volume: + name: "{{ web_app_name }}_data" + state: absent +``` + +**1. What are the security implications of storing SSH keys in GitHub Secrets?** +A: +- **Pros:** Encrypted at rest, access-controlled, audit logged +- **Cons:** GitHub has access, potential for compromise if GitHub is breached +- **Best Practice:** Use dedicated deployment keys with minimal permissions, rotate regularly +- **Alternative:** Use GitHub's OIDC for keyless authentication + +**2. How would you implement a staging β†’ production deployment pipeline?** +A: +```yaml +jobs: + deploy-staging: + # Deploy to staging + + manual-approval: + needs: deploy-staging + environment: production # Requires manual approval + + deploy-production: + needs: manual-approval + # Deploy to production +``` + +**3. What would you add to make rollbacks possible?** +A: +- Tag Docker images with git commit SHA +- Store previous deployment state +- Create rollback playbook that deploys previous version +- Add workflow_dispatch input for version selection +- Keep deployment history in artifact storage + +**4. How does self-hosted runner improve security compared to GitHub-hosted?** +A: +- No SSH keys needed (runner is on target network) +- Secrets never leave your infrastructure +- Full control over runner environment +- Can use internal DNS/networks +- Reduced attack surface (no internet-exposed SSH) \ No newline at end of file diff --git a/ansible/docs/image-1.png b/ansible/docs/image-1.png new file mode 100644 index 0000000000..957f64c6aa Binary files /dev/null and b/ansible/docs/image-1.png differ diff --git a/ansible/docs/image.png b/ansible/docs/image.png new file mode 100644 index 0000000000..e70926742b Binary files /dev/null and b/ansible/docs/image.png differ diff --git a/ansible/group_vars/all.yml b/ansible/group_vars/all.yml new file mode 100644 index 0000000000..8aad52c722 --- /dev/null +++ b/ansible/group_vars/all.yml @@ -0,0 +1,40 @@ +$ANSIBLE_VAULT;1.1;AES256 +62636562336237353030636134373563326531313533333536353365356131656234613561663166 +3862636333626536316430326232663633646561333931360a346261613362313834633936633439 +66333965316561306535663164643031393239386664623365616363346162623634633063373961 +6231623833613639340a386436663237376130376331353436333139623335353661343437303136 +36336633656362623463386461366637343533343731653366633537373066653532393863373632 +36333461613364316436306534636231333663643738623530626532383664663633383632633934 +61333265336433323562666561393362633066306138346335663563386262343436316235356435 +34303830323032653366363336643531363339663538373761396632313635356231333838353432 +30663965643965353662656139356465336261396463656462646537663334663632313637643931 +30663538386535313734353635343735303463366336363336636463373234653539316630346538 +37626239386633643632376433643136623830356636353935636530373763366434636637346433 +35353237353936613738303666373537323239393334656437363666363735383362643361356234 +64366530656162316633396565393964633932336162393564373464663064313637306539343134 +37373034663039333133323934346566643939343632396334363031366238376137363938333066 +32653161623361326332643262656130646638383537663230356266373630313163663436623365 +30336466333661653164336630363935636433376564613530643966363035353234323034386261 +63636439626338666238356362383339636165386438373133393036383733303165343234346638 +39323230653333383834383062666165336435316339643831336432623765333936346266303064 +38313738623761616564303363356438633764633434356131383338663434643063396261306539 +36656461386664336335613963343364613264636161393063633762393965366536626639323533 +63656466353836653263386261653035333465313535366266333561313532396264343039343363 +33323631336234393030656561363532396265666661326538653464616532356437313464653737 +64336537633861323032313136373861303335363434633530626136333936316562353732646635 +39663033653266303434356266393531386138623435323037313462643562306633323430383333 +35643135633765363064316161333663643236613964356265613861373731346633333262313037 +63393034323632376239313565626464633433663666376263393064656138373437333539396338 +37373362613137353938656164616364373930303639636534626134643237613532303933653336 +38653639626565633838646531313265383039633438616239646262316266333134396538343533 +31646638613064613731356636646566383737646135383166613338666364363265313633623063 +66366638373233373364626130383238363732363039356235323739383662313136373965393233 +31303234663365363930333435613064346462656437373563373635323332353235656532656434 +37393665333430343737633636616439616439326538636439616331383663363232383339653031 +39306233643535316134356432643234613238353132353537353333323264316535653634363661 +61383663633931663134376463316466346437613661333465666366353834306432623632303663 +35323561646335356233323962386238336635393831333461303365663936376362316266353763 +37316362386637386330313831386136653063386638306133616237663766396263326235653031 +37623263383462396134393535313438346631346532656231663334313136396661623238363132 +39616532633165616265633163303765633534616165363034666434623536373264336362313630 +396561663630303135303330366166663232 diff --git a/ansible/inventory/hosts.ini b/ansible/inventory/hosts.ini new file mode 100644 index 0000000000..e11153cdeb --- /dev/null +++ b/ansible/inventory/hosts.ini @@ -0,0 +1,5 @@ +[webservers] +lab04-vm ansible_host=93.77.180.155 ansible_user=ubuntu ansible_ssh_private_key_file=~/.ssh/yandex_cloud_key + +[webservers:vars] +ansible_python_interpreter=/usr/bin/python3 diff --git a/ansible/playbooks/deploy-monitoring.yml b/ansible/playbooks/deploy-monitoring.yml new file mode 100644 index 0000000000..74ea6a2926 --- /dev/null +++ b/ansible/playbooks/deploy-monitoring.yml @@ -0,0 +1,32 @@ +--- +# Playbook to deploy monitoring stack with Loki, Promtail, and Grafana + +- name: Deploy Monitoring Stack + hosts: localhost + become: false + gather_facts: true + + roles: + - monitoring + + post_tasks: + - name: Display access information + ansible.builtin.debug: + msg: | + ======================================== + Monitoring Stack Deployed Successfully! + ======================================== + + Grafana UI: http://localhost:3000 + Username: admin + Password: {{ grafana_admin_password }} + + Loki API: http://localhost:3100 + + Next Steps: + 1. Open Grafana in your browser + 2. Add Loki data source (http://loki:3100) + 3. Explore logs in the Explore tab + 4. Create dashboards for your applications + + ======================================== \ No newline at end of file diff --git a/ansible/playbooks/deploy.yml b/ansible/playbooks/deploy.yml new file mode 100644 index 0000000000..95174b9e0e --- /dev/null +++ b/ansible/playbooks/deploy.yml @@ -0,0 +1,7 @@ +--- +- name: Deploy application + hosts: webservers + become: true + + roles: + - web_app diff --git a/ansible/playbooks/deploy_python.yml b/ansible/playbooks/deploy_python.yml new file mode 100644 index 0000000000..d9b839a229 --- /dev/null +++ b/ansible/playbooks/deploy_python.yml @@ -0,0 +1,9 @@ +--- +- name: Deploy Python Application + hosts: webservers + become: true + vars_files: + - ../vars/app_python.yml + + roles: + - web_app diff --git a/ansible/playbooks/provision.yml b/ansible/playbooks/provision.yml new file mode 100644 index 0000000000..17d437513f --- /dev/null +++ b/ansible/playbooks/provision.yml @@ -0,0 +1,8 @@ +--- +- name: Provision web servers + hosts: webservers + become: yes + + roles: + - common + - docker \ No newline at end of file diff --git a/ansible/playbooks/site.yml b/ansible/playbooks/site.yml new file mode 100644 index 0000000000..e1c07df503 --- /dev/null +++ b/ansible/playbooks/site.yml @@ -0,0 +1,9 @@ +--- +- name: Complete infrastructure setup and deployment + hosts: webservers + become: yes + + roles: + - common + - docker + - app_deploy \ No newline at end of file diff --git a/ansible/playbooks/test_rescue.yml b/ansible/playbooks/test_rescue.yml new file mode 100644 index 0000000000..14925b8908 --- /dev/null +++ b/ansible/playbooks/test_rescue.yml @@ -0,0 +1,53 @@ +--- +- name: Test rescue block in common role + hosts: webservers + become: true + + tasks: + - name: Temporarily break apt sources + block: + - name: Backup original sources.list + copy: + src: /etc/apt/sources.list + dest: /etc/apt/sources.list.backup + remote_src: yes + + - name: Add invalid repository to trigger error + lineinfile: + path: /etc/apt/sources.list + line: "deb http://invalid-repo.example.com/ubuntu focal main" + state: present + + - name: Try to update apt cache (will fail) + apt: + update_cache: yes + register: apt_result + + rescue: + - name: Rescue block activated! + debug: + msg: "RESCUE BLOCK TRIGGERED! Fixing apt sources..." + + - name: Restore original sources.list + copy: + src: /etc/apt/sources.list.backup + dest: /etc/apt/sources.list + remote_src: yes + + - name: Run apt-get update --fix-missing + command: apt-get update --fix-missing + changed_when: true + + - name: Retry apt update + apt: + update_cache: yes + + always: + - name: Cleanup backup file + file: + path: /etc/apt/sources.list.backup + state: absent + + - name: Always block executed + debug: + msg: "Always block runs regardless of success/failure" \ No newline at end of file diff --git a/ansible/roles/common/defaults/main.yml b/ansible/roles/common/defaults/main.yml new file mode 100644 index 0000000000..b749bbdf47 --- /dev/null +++ b/ansible/roles/common/defaults/main.yml @@ -0,0 +1,25 @@ +--- +# Common role - Default variables + +common_packages: + - python3-pip + - curl + - git + - vim + - htop + - wget + - unzip + - software-properties-common + - apt-transport-https + - ca-certificates + - gnupg + - lsb-release + +timezone: "UTC" + +# User management +deploy_user: "deploy" +deploy_user_groups: + - sudo +deploy_user_shell: "/bin/bash" +create_deploy_user: false # Set to true to create deployment user \ No newline at end of file diff --git a/ansible/roles/common/tasks/main.yml b/ansible/roles/common/tasks/main.yml new file mode 100644 index 0000000000..73f9e0fec4 --- /dev/null +++ b/ansible/roles/common/tasks/main.yml @@ -0,0 +1,95 @@ +--- +# Common role - System provisioning tasks + +# Package installation block with error handling +- name: Package installation tasks + block: + - name: Update apt cache + apt: + update_cache: yes + cache_valid_time: 3600 + + - name: Install common packages + apt: + name: "{{ common_packages }}" + state: present + update_cache: yes + + rescue: + - name: Handle apt cache update failure + debug: + msg: "Apt cache update failed, attempting fix..." + + - name: Fix missing packages + command: apt-get update --fix-missing + changed_when: true + + - name: Retry package installation + apt: + name: "{{ common_packages }}" + state: present + update_cache: yes + + always: + - name: Log package installation completion + copy: + content: "Common packages installation completed at {{ ansible_date_time.iso8601 }}\n" + dest: /tmp/ansible_common_packages.log + mode: '0644' + + become: true + tags: + - common + - packages + +# User management block +- name: User management tasks + block: + - name: Create deployment user + user: + name: "{{ deploy_user }}" + groups: "{{ deploy_user_groups }}" + shell: "{{ deploy_user_shell }}" + create_home: yes + append: yes + when: create_deploy_user | bool + + - name: Set up SSH directory for deployment user + file: + path: "/home/{{ deploy_user }}/.ssh" + state: directory + owner: "{{ deploy_user }}" + group: "{{ deploy_user }}" + mode: '0700' + when: create_deploy_user | bool + + always: + - name: Log user management completion + copy: + content: "User management completed at {{ ansible_date_time.iso8601 }}\n" + dest: /tmp/ansible_common_users.log + mode: '0644' + + become: true + tags: + - common + - users + +# System configuration block +- name: System configuration tasks + block: + - name: Set timezone + community.general.timezone: + name: "{{ timezone }}" + + always: + - name: Log system configuration completion + copy: + content: "System configuration completed at {{ ansible_date_time.iso8601 }}\n" + dest: /tmp/ansible_common_config.log + mode: '0644' + + become: true + tags: + - common + - config \ No newline at end of file diff --git a/ansible/roles/docker/defaults/main.yml b/ansible/roles/docker/defaults/main.yml new file mode 100644 index 0000000000..0eaf0e6ae3 --- /dev/null +++ b/ansible/roles/docker/defaults/main.yml @@ -0,0 +1,5 @@ +--- +# Docker role - Default variables + +docker_user: "ubuntu" +docker_version: "latest" \ No newline at end of file diff --git a/ansible/roles/docker/handlers/main.yml b/ansible/roles/docker/handlers/main.yml new file mode 100644 index 0000000000..e7f0b8f7f2 --- /dev/null +++ b/ansible/roles/docker/handlers/main.yml @@ -0,0 +1,10 @@ +--- +# Docker role - Handlers + +- name: restart docker + service: + name: docker + state: restarted + tags: + - docker + - service \ No newline at end of file diff --git a/ansible/roles/docker/tasks/main.yml b/ansible/roles/docker/tasks/main.yml new file mode 100644 index 0000000000..3f984636de --- /dev/null +++ b/ansible/roles/docker/tasks/main.yml @@ -0,0 +1,136 @@ +--- +# Docker role - Docker installation tasks + +# Docker installation block with error handling +- name: Docker installation tasks + block: + - name: Install prerequisites for Docker + apt: + name: + - apt-transport-https + - ca-certificates + - curl + - gnupg + - lsb-release + state: present + update_cache: yes + + - name: Create directory for Docker GPG key + file: + path: /etc/apt/keyrings + state: directory + mode: '0755' + + - name: Add Docker GPG key + apt_key: + url: https://download.docker.com/linux/ubuntu/gpg + keyring: /etc/apt/keyrings/docker.gpg + state: present + + - name: Get Ubuntu release codename + command: lsb_release -cs + register: ubuntu_release + changed_when: false + + - name: Get system architecture + command: dpkg --print-architecture + register: system_arch + changed_when: false + + - name: Add Docker repository + apt_repository: + repo: "deb [arch={{ system_arch.stdout }} signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/ubuntu {{ ubuntu_release.stdout }} stable" + state: present + filename: docker + notify: restart docker + + - name: Install Docker packages + apt: + name: + - docker-ce + - docker-ce-cli + - containerd.io + - docker-buildx-plugin + - docker-compose-plugin + state: present + update_cache: yes + notify: restart docker + + - name: Install python3-docker for Ansible docker modules + apt: + name: python3-docker + state: present + + rescue: + - name: Handle Docker installation failure + debug: + msg: "Docker installation failed, retrying GPG key addition..." + + - name: Wait before retry + wait_for: + timeout: 10 + + - name: Retry Docker GPG key addition + apt_key: + url: https://download.docker.com/linux/ubuntu/gpg + keyring: /etc/apt/keyrings/docker.gpg + state: present + + - name: Retry Docker package installation + apt: + name: + - docker-ce + - docker-ce-cli + - containerd.io + - docker-buildx-plugin + - docker-compose-plugin + state: present + update_cache: yes + + always: + - name: Ensure Docker service is enabled and started + service: + name: docker + state: started + enabled: yes + + - name: Log Docker installation completion + copy: + content: "Docker installation completed at {{ ansible_date_time.iso8601 }}\n" + dest: /tmp/ansible_docker_install.log + mode: '0644' + + become: true + tags: + - docker + - docker_install + +# Docker configuration block +- name: Docker configuration tasks + block: + - name: Add user to docker group + user: + name: "{{ docker_user }}" + groups: docker + append: yes + + - name: Verify Docker installation + command: docker --version + register: docker_version + changed_when: false + + - name: Display Docker version + debug: + msg: "Docker version installed: {{ docker_version.stdout }}" + + always: + - name: Log Docker configuration completion + copy: + content: "Docker configuration completed at {{ ansible_date_time.iso8601 }}\n" + dest: /tmp/ansible_docker_config.log + mode: '0644' + + become: true + tags: + - docker + - docker_config \ No newline at end of file diff --git a/ansible/roles/monitoring/defaults/main.yml b/ansible/roles/monitoring/defaults/main.yml new file mode 100644 index 0000000000..c262302966 --- /dev/null +++ b/ansible/roles/monitoring/defaults/main.yml @@ -0,0 +1,44 @@ +--- +# Monitoring role default variables + +# Service versions +loki_version: "3.0.0" +promtail_version: "3.0.0" +grafana_version: "12.3.1" + +# Ports +loki_port: 3100 +grafana_port: 3000 +promtail_port: 9080 + +# Retention +log_retention_period: "168h" # 7 days + +# Resource limits +loki_cpu_limit: "1.0" +loki_memory_limit: "1G" +loki_cpu_reservation: "0.5" +loki_memory_reservation: "512M" + +promtail_cpu_limit: "0.5" +promtail_memory_limit: "512M" +promtail_cpu_reservation: "0.25" +promtail_memory_reservation: "256M" + +grafana_cpu_limit: "1.0" +grafana_memory_limit: "1G" +grafana_cpu_reservation: "0.5" +grafana_memory_reservation: "512M" + +# Grafana settings +grafana_admin_password: "admin123" +grafana_anonymous_enabled: false + +# Loki schema version +loki_schema_version: "v13" + +# Monitoring directory +monitoring_dir: "/opt/monitoring" + +# Docker Compose version +docker_compose_version: "3.8" \ No newline at end of file diff --git a/ansible/roles/monitoring/handlers/main.yml b/ansible/roles/monitoring/handlers/main.yml new file mode 100644 index 0000000000..1eee523e87 --- /dev/null +++ b/ansible/roles/monitoring/handlers/main.yml @@ -0,0 +1,7 @@ +--- +# Handlers for monitoring role + +- name: Restart monitoring stack + community.docker.docker_compose_v2: + project_src: "{{ monitoring_dir }}" + state: restarted \ No newline at end of file diff --git a/ansible/roles/monitoring/meta/main.yml b/ansible/roles/monitoring/meta/main.yml new file mode 100644 index 0000000000..6099501f8a --- /dev/null +++ b/ansible/roles/monitoring/meta/main.yml @@ -0,0 +1,2 @@ +--- +dependencies: [] \ No newline at end of file diff --git a/ansible/roles/monitoring/tasks/deploy.yml b/ansible/roles/monitoring/tasks/deploy.yml new file mode 100644 index 0000000000..c47522e69a --- /dev/null +++ b/ansible/roles/monitoring/tasks/deploy.yml @@ -0,0 +1,35 @@ +--- +# Deployment tasks for monitoring stack + +- name: Deploy monitoring stack with Docker Compose + community.docker.docker_compose_v2: + project_src: "{{ monitoring_dir }}" + state: present + pull: always + register: monitoring_deploy + +- name: Wait for Loki to be ready + ansible.builtin.uri: + url: "http://localhost:{{ loki_port }}/ready" + status_code: 200 + register: loki_ready + until: loki_ready.status == 200 + retries: 30 + delay: 2 + +- name: Wait for Grafana to be ready + ansible.builtin.uri: + url: "http://localhost:{{ grafana_port }}/api/health" + status_code: 200 + register: grafana_ready + until: grafana_ready.status == 200 + retries: 30 + delay: 2 + +- name: Display deployment status + ansible.builtin.debug: + msg: | + Monitoring stack deployed successfully! + - Loki: http://localhost:{{ loki_port }} + - Grafana: http://localhost:{{ grafana_port }} + - Grafana credentials: admin / {{ grafana_admin_password }} \ No newline at end of file diff --git a/ansible/roles/monitoring/tasks/main.yml b/ansible/roles/monitoring/tasks/main.yml new file mode 100644 index 0000000000..0e213be217 --- /dev/null +++ b/ansible/roles/monitoring/tasks/main.yml @@ -0,0 +1,14 @@ +--- +# Main tasks for monitoring role + +- name: Include setup tasks + ansible.builtin.include_tasks: setup.yml + tags: + - monitoring + - setup + +- name: Include deployment tasks + ansible.builtin.include_tasks: deploy.yml + tags: + - monitoring + - deploy \ No newline at end of file diff --git a/ansible/roles/monitoring/tasks/setup.yml b/ansible/roles/monitoring/tasks/setup.yml new file mode 100644 index 0000000000..eb8c795b36 --- /dev/null +++ b/ansible/roles/monitoring/tasks/setup.yml @@ -0,0 +1,41 @@ +--- +# Setup tasks for monitoring stack + +- name: Create monitoring directory structure + ansible.builtin.file: + path: "{{ item }}" + state: directory + mode: '0755' + loop: + - "{{ monitoring_dir }}" + - "{{ monitoring_dir }}/loki" + - "{{ monitoring_dir }}/promtail" + - "{{ monitoring_dir }}/docs" + +- name: Template Loki configuration + ansible.builtin.template: + src: loki-config.yml.j2 + dest: "{{ monitoring_dir }}/loki/config.yml" + mode: '0644' + notify: Restart monitoring stack + +- name: Template Promtail configuration + ansible.builtin.template: + src: promtail-config.yml.j2 + dest: "{{ monitoring_dir }}/promtail/config.yml" + mode: '0644' + notify: Restart monitoring stack + +- name: Template Docker Compose file + ansible.builtin.template: + src: docker-compose.yml.j2 + dest: "{{ monitoring_dir }}/docker-compose.yml" + mode: '0644' + notify: Restart monitoring stack + +- name: Create .env file for secrets + ansible.builtin.template: + src: env.j2 + dest: "{{ monitoring_dir }}/.env" + mode: '0600' + no_log: true \ No newline at end of file diff --git a/ansible/roles/monitoring/templates/docker-compose.yml.j2 b/ansible/roles/monitoring/templates/docker-compose.yml.j2 new file mode 100644 index 0000000000..c734ac952a --- /dev/null +++ b/ansible/roles/monitoring/templates/docker-compose.yml.j2 @@ -0,0 +1,98 @@ +version: '{{ docker_compose_version }}' + +services: + loki: + image: grafana/loki:{{ loki_version }} + container_name: loki + ports: + - "{{ loki_port }}:{{ loki_port }}" + volumes: + - ./loki/config.yml:/etc/loki/config.yml + - loki-data:/loki + command: -config.file=/etc/loki/config.yml + networks: + - logging + labels: + logging: "promtail" + app: "loki" + healthcheck: + test: ["CMD-SHELL", "wget --no-verbose --tries=1 --spider http://localhost:{{ loki_port }}/ready || exit 1"] + interval: 10s + timeout: 5s + retries: 5 + start_period: 10s + deploy: + resources: + limits: + cpus: '{{ loki_cpu_limit }}' + memory: {{ loki_memory_limit }} + reservations: + cpus: '{{ loki_cpu_reservation }}' + memory: {{ loki_memory_reservation }} + + promtail: + image: grafana/promtail:{{ promtail_version }} + container_name: promtail + ports: + - "{{ promtail_port }}:{{ promtail_port }}" + volumes: + - ./promtail/config.yml:/etc/promtail/config.yml + - /var/lib/docker/containers:/var/lib/docker/containers:ro + - /var/run/docker.sock:/var/run/docker.sock:ro + command: -config.file=/etc/promtail/config.yml + networks: + - logging + labels: + logging: "promtail" + app: "promtail" + depends_on: + - loki + deploy: + resources: + limits: + cpus: '{{ promtail_cpu_limit }}' + memory: {{ promtail_memory_limit }} + reservations: + cpus: '{{ promtail_cpu_reservation }}' + memory: {{ promtail_memory_reservation }} + + grafana: + image: grafana/grafana:{{ grafana_version }} + container_name: grafana + ports: + - "{{ grafana_port }}:{{ grafana_port }}" + volumes: + - grafana-data:/var/lib/grafana + environment: + - GF_AUTH_ANONYMOUS_ENABLED={{ grafana_anonymous_enabled | lower }} + - GF_SECURITY_ADMIN_PASSWORD=${GRAFANA_ADMIN_PASSWORD} + - GF_SECURITY_ALLOW_EMBEDDING=true + networks: + - logging + labels: + logging: "promtail" + app: "grafana" + depends_on: + - loki + healthcheck: + test: ["CMD-SHELL", "wget --no-verbose --tries=1 --spider http://localhost:{{ grafana_port }}/api/health || exit 1"] + interval: 10s + timeout: 5s + retries: 5 + start_period: 10s + deploy: + resources: + limits: + cpus: '{{ grafana_cpu_limit }}' + memory: {{ grafana_memory_limit }} + reservations: + cpus: '{{ grafana_cpu_reservation }}' + memory: {{ grafana_memory_reservation }} + +networks: + logging: + driver: bridge + +volumes: + loki-data: + grafana-data: \ No newline at end of file diff --git a/ansible/roles/monitoring/templates/env.j2 b/ansible/roles/monitoring/templates/env.j2 new file mode 100644 index 0000000000..fc8415aae2 --- /dev/null +++ b/ansible/roles/monitoring/templates/env.j2 @@ -0,0 +1 @@ +GRAFANA_ADMIN_PASSWORD={{ grafana_admin_password }} \ No newline at end of file diff --git a/ansible/roles/monitoring/templates/loki-config.yml.j2 b/ansible/roles/monitoring/templates/loki-config.yml.j2 new file mode 100644 index 0000000000..bb0c0eb45c --- /dev/null +++ b/ansible/roles/monitoring/templates/loki-config.yml.j2 @@ -0,0 +1,75 @@ +auth_enabled: false + +server: + http_listen_port: {{ loki_port }} + grpc_listen_port: 9096 + +common: + instance_addr: 127.0.0.1 + path_prefix: /loki + storage: + filesystem: + chunks_directory: /loki/chunks + rules_directory: /loki/rules + replication_factor: 1 + ring: + kvstore: + store: inmemory + +query_range: + results_cache: + cache: + embedded_cache: + enabled: true + max_size_mb: 100 + +schema_config: + configs: + - from: 2024-01-01 + store: tsdb + object_store: filesystem + schema: {{ loki_schema_version }} + index: + prefix: index_ + period: 24h + +storage_config: + tsdb_shipper: + active_index_directory: /loki/tsdb-index + cache_location: /loki/tsdb-cache + cache_ttl: 24h + filesystem: + directory: /loki/chunks + +limits_config: + retention_period: {{ log_retention_period }} + reject_old_samples: true + reject_old_samples_max_age: {{ log_retention_period }} + ingestion_rate_mb: 10 + ingestion_burst_size_mb: 20 + max_query_series: 500 + max_query_parallelism: 32 + +compactor: + working_directory: /loki/compactor + compaction_interval: 10m + retention_enabled: true + retention_delete_delay: 2h + retention_delete_worker_count: 150 + delete_request_store: filesystem + +table_manager: + retention_deletes_enabled: true + retention_period: {{ log_retention_period }} + +ruler: + storage: + type: local + local: + directory: /loki/rules + rule_path: /loki/rules-temp + alertmanager_url: http://localhost:9093 + ring: + kvstore: + store: inmemory + enable_api: true \ No newline at end of file diff --git a/ansible/roles/monitoring/templates/promtail-config.yml.j2 b/ansible/roles/monitoring/templates/promtail-config.yml.j2 new file mode 100644 index 0000000000..fa7f5f345f --- /dev/null +++ b/ansible/roles/monitoring/templates/promtail-config.yml.j2 @@ -0,0 +1,63 @@ +server: + http_listen_port: {{ promtail_port }} + grpc_listen_port: 0 + +positions: + filename: /tmp/positions.yaml + +clients: + - url: http://loki:{{ loki_port }}/loki/api/v1/push + +scrape_configs: + - job_name: docker + docker_sd_configs: + - host: unix:///var/run/docker.sock + refresh_interval: 5s + filters: + - name: label + values: ["logging=promtail"] + relabel_configs: + # Extract container name + - source_labels: ['__meta_docker_container_name'] + regex: '/(.*)' + target_label: 'container' + # Extract container ID + - source_labels: ['__meta_docker_container_id'] + target_label: 'container_id' + # Extract image name + - source_labels: ['__meta_docker_container_label_com_docker_compose_service'] + target_label: 'service' + # Extract app label if present + - source_labels: ['__meta_docker_container_label_app'] + target_label: 'app' + # Add job label + - source_labels: ['__meta_docker_container_label_logging'] + target_label: 'job' + replacement: 'docker' + + # Pipeline stages to parse JSON logs + pipeline_stages: + # Parse JSON from log line + - json: + expressions: + level: level + message: message + timestamp: timestamp + logger: logger + service: service + method: method + path: path + status_code: status_code + client_ip: client_ip + + # Add extracted fields as labels + - labels: + level: + service: + method: + status_code: + + # Use timestamp from log if available + - timestamp: + source: timestamp + format: RFC3339Nano \ No newline at end of file diff --git a/ansible/roles/web_app/defaults/main.yml b/ansible/roles/web_app/defaults/main.yml new file mode 100644 index 0000000000..7b05042833 --- /dev/null +++ b/ansible/roles/web_app/defaults/main.yml @@ -0,0 +1,26 @@ +--- +# Default variables for web_app role + +# Application Configuration +web_app_name: devops-app +web_app_docker_image: "{{ dockerhub_username }}/devops-info-service" +web_app_docker_tag: latest +web_app_port: 8000 +web_app_internal_port: 8000 + +# Docker Compose Configuration +web_app_compose_project_dir: "/opt/{{ web_app_name }}" +web_app_docker_compose_version: "3.8" + +# Wipe Logic Control +web_app_wipe: false # Default: do not wipe + +# Set to true to remove application completely +# Wipe only: ansible-playbook deploy.yml -e "web_app_wipe=true" --tags web_app_wipe +# Clean install: ansible-playbook deploy.yml -e "web_app_wipe=true" + +# Application restart policy +web_app_restart_policy: unless-stopped + +# Environment variables (can be overridden in group_vars or host_vars) +web_app_environment_vars: {} diff --git a/ansible/roles/web_app/handlers/main.yml b/ansible/roles/web_app/handlers/main.yml new file mode 100644 index 0000000000..e2be26642f --- /dev/null +++ b/ansible/roles/web_app/handlers/main.yml @@ -0,0 +1,11 @@ +--- +# Application deployment role - Handlers + +- name: Restart application + community.docker.docker_container: + name: "{{ app_container_name }}" + state: started + restart: true + tags: + - app + - service diff --git a/ansible/roles/web_app/meta/main.yml b/ansible/roles/web_app/meta/main.yml new file mode 100644 index 0000000000..96a77aa2ac --- /dev/null +++ b/ansible/roles/web_app/meta/main.yml @@ -0,0 +1,11 @@ +--- +# Role dependencies for web_app +# This ensures Docker is installed before deploying the web application + +dependencies: + - role: docker + # Docker role must be executed first to ensure: + # - Docker engine is installed + # - Docker Compose plugin is available + # - Docker service is running + # - User has proper permissions diff --git a/ansible/roles/web_app/tasks/main.yml b/ansible/roles/web_app/tasks/main.yml new file mode 100644 index 0000000000..6cc77e4dd8 --- /dev/null +++ b/ansible/roles/web_app/tasks/main.yml @@ -0,0 +1,98 @@ +--- +# Web application deployment role - Deploy with Docker Compose + +# Wipe logic runs first (when explicitly requested) +- name: Include wipe tasks + ansible.builtin.include_tasks: wipe.yml + tags: + - web_app_wipe + +# Deployment tasks follow +- name: Deploy application with Docker Compose + become: true + tags: + - app_deploy + - compose + block: + - name: Check if application is already running + ansible.builtin.command: docker ps --filter "name={{ web_app_name }}" --format "{{ '{{' }}.Names{{ '}}' }}" + register: web_app_running_containers + changed_when: false + failed_when: false + + - name: Display current container status + ansible.builtin.debug: + msg: "Container {{ web_app_name }} is {{ 'running' if web_app_name in web_app_running_containers.stdout else 'not running' }}" + + - name: Create application directory + ansible.builtin.file: + path: "{{ web_app_compose_project_dir }}" + state: directory + mode: '0755' + + - name: Template docker-compose file + ansible.builtin.template: + src: docker-compose.yml.j2 + dest: "{{ web_app_compose_project_dir }}/docker-compose.yml" + mode: '0644' + register: web_app_compose_template + + - name: Log in to Docker Hub + community.docker.docker_login: + username: "{{ dockerhub_username }}" + password: "{{ dockerhub_password }}" + registry_url: https://index.docker.io/v1/ + no_log: true + + - name: Deploy with Docker Compose + community.docker.docker_compose_v2: + project_src: "{{ web_app_compose_project_dir }}" + state: present + pull: always + recreate: auto + register: web_app_compose_result + + - name: Display deployment result + ansible.builtin.debug: + msg: >- + Deployment {{ 'changed' if web_app_compose_result.changed else 'unchanged' }} - + containers are {{ 'up to date' if not web_app_compose_result.changed else 'updated' }} + + - name: Wait for application port to be available (on target VM) + ansible.builtin.wait_for: + host: 127.0.0.1 + port: "{{ web_app_port }}" + delay: 5 + timeout: 60 + state: started + delegate_to: "{{ inventory_hostname }}" + + - name: Verify health endpoint (from target VM) + ansible.builtin.uri: + url: "http://127.0.0.1:{{ web_app_port }}/health" + method: GET + status_code: 200 + timeout: 10 + register: web_app_health_check + retries: 3 + delay: 5 + until: web_app_health_check.status == 200 + delegate_to: "{{ inventory_hostname }}" + + - name: Display health check result + ansible.builtin.debug: + msg: "Application is healthy: {{ web_app_health_check.json }}" + + rescue: + - name: Handle deployment failure + ansible.builtin.debug: + msg: "Deployment failed. Check logs with: docker compose -f {{ web_app_compose_project_dir }}/docker-compose.yml logs" + + - name: Display container status + ansible.builtin.command: docker ps -a + register: web_app_container_status + changed_when: false + + - name: Show container status + ansible.builtin.debug: + var: web_app_container_status.stdout_lines diff --git a/ansible/roles/web_app/tasks/wipe.yml b/ansible/roles/web_app/tasks/wipe.yml new file mode 100644 index 0000000000..11b9fbcdbf --- /dev/null +++ b/ansible/roles/web_app/tasks/wipe.yml @@ -0,0 +1,44 @@ +--- +# Wipe logic for web application +# This file contains tasks to completely remove the deployed application + +- name: Wipe web application + when: web_app_wipe | bool + become: true + tags: + - web_app_wipe + block: + - name: Check if docker-compose file exists + ansible.builtin.stat: + path: "{{ web_app_compose_project_dir }}/docker-compose.yml" + register: web_app_compose_file + + - name: Stop and remove containers with Docker Compose + community.docker.docker_compose_v2: + project_src: "{{ web_app_compose_project_dir }}" + state: absent + when: web_app_compose_file.stat.exists + failed_when: false + + - name: Stop container manually if compose file doesn't exist + community.docker.docker_container: + name: "{{ web_app_name }}" + state: absent + when: not web_app_compose_file.stat.exists + failed_when: false + + - name: Remove docker-compose file + ansible.builtin.file: + path: "{{ web_app_compose_project_dir }}/docker-compose.yml" + state: absent + failed_when: false + + - name: Remove application directory + ansible.builtin.file: + path: "{{ web_app_compose_project_dir }}" + state: absent + failed_when: false + + - name: Log wipe completion + ansible.builtin.debug: + msg: "Application {{ web_app_name }} wiped successfully from {{ web_app_compose_project_dir }}" diff --git a/ansible/roles/web_app/templates/docker-compose.yml.j2 b/ansible/roles/web_app/templates/docker-compose.yml.j2 new file mode 100644 index 0000000000..2d0059352a --- /dev/null +++ b/ansible/roles/web_app/templates/docker-compose.yml.j2 @@ -0,0 +1,19 @@ +services: + {{ web_app_name }}: + image: {{ web_app_docker_image }}:{{ web_app_docker_tag }} + container_name: {{ web_app_name }} + ports: + - "{{ web_app_port }}:{{ web_app_internal_port }}" +{% if web_app_environment_vars is defined and web_app_environment_vars %} + environment: +{% for key, value in web_app_environment_vars.items() %} + {{ key }}: "{{ value }}" +{% endfor %} +{% endif %} + restart: {{ web_app_restart_policy }} + networks: + - app_network + +networks: + app_network: + driver: bridge \ No newline at end of file diff --git a/ansible/vars/app_python.yml b/ansible/vars/app_python.yml new file mode 100644 index 0000000000..86b1b08025 --- /dev/null +++ b/ansible/vars/app_python.yml @@ -0,0 +1,14 @@ +--- +# Python application variables + +web_app_name: devops-python +web_app_docker_image: "{{ dockerhub_username }}/python_app" +web_app_docker_tag: latest +web_app_port: 8000 +web_app_internal_port: 8000 +web_app_compose_project_dir: "/opt/{{ web_app_name }}" + +# Application-specific environment variables +web_app_environment_vars: + APP_NAME: "DevOps Python Service" + APP_VERSION: "1.0.0" \ No newline at end of file diff --git a/app_go/.dockerignore b/app_go/.dockerignore new file mode 100644 index 0000000000..52c65fe81d --- /dev/null +++ b/app_go/.dockerignore @@ -0,0 +1,9 @@ +.git +*.md +docs/ +bin/ +dist/ +tmp/ +.idea/ +.vscode/ +.DS_Store \ No newline at end of file diff --git a/app_go/.gitignore b/app_go/.gitignore new file mode 100644 index 0000000000..d6c75d54e3 --- /dev/null +++ b/app_go/.gitignore @@ -0,0 +1,18 @@ +# Go binaries / build output +devops-info-service +*.exe +*.out +*.test +bin/ +dist/ + +# IDE/editor +.vscode/ +.idea/ + +# OS files +.DS_Store +Thumbs.db + +# Logs +*.log \ No newline at end of file diff --git a/app_go/Dockerfile b/app_go/Dockerfile new file mode 100644 index 0000000000..6dd906a964 --- /dev/null +++ b/app_go/Dockerfile @@ -0,0 +1,14 @@ +# πŸ”¨ Stage 1: Builder +FROM golang:1.22-alpine AS builder +WORKDIR /app +COPY go.mod ./ +RUN go mod download +COPY . . +RUN CGO_ENABLED=0 go build -o myapp + +# πŸš€ Stage 2: Runtime +FROM gcr.io/distroless/static-debian12:nonroot +WORKDIR /app +COPY --from=builder /app/myapp . +EXPOSE 8080 +CMD ["./myapp"] \ No newline at end of file diff --git a/app_go/README.md b/app_go/README.md new file mode 100644 index 0000000000..f2870f9688 --- /dev/null +++ b/app_go/README.md @@ -0,0 +1,47 @@ +[![Go CI](https://github.com/newspec/DevOps-Core-Course/actions/workflows/go-ci.yml/badge.svg?branch=lab03)](https://github.com/newspec/DevOps-Core-Course/actions/workflows/go-ci.yml?query=branch%3Alab03) +[![Coverage](https://codecov.io/gh/newspec/DevOps-Core-Course/branch/lab03/graph/badge.svg?flag=go)](https://codecov.io/gh/newspec/DevOps-Core-Course/branch/lab03?flag=go) + + +# devops-info-service (Go) + +## Overview +`devops-info-service` is a lightweight HTTP service written in Go. It returns: +- service metadata (name, version, description, framework), +- system information (hostname, OS/platform, architecture, CPU count, Go version), +- runtime information (uptime, current UTC time), +- request information (client IP, user-agent, method, path), +- a list of available endpoints. + +This is useful for DevOps labs and basic observability: quick environment inspection and health checks. + +--- + +## Prerequisites +- **Go:** 1.22+ (recommended) +- No external dependencies (standard library only) + +--- + +## Installation +```bash +cd app_go +go mod tidy +``` + +## Running the Application +```bash +go run . +``` + +## API Endpoints +- `GET /` - Service and system information +- `GET /health` - Health check + +## Configuration + +The application is configured using environment variables. + +| Variable | Default | Description | Example | +|---------|---------|-------------|---------| +| `HOST` | `0.0.0.0` | Host interface to bind the server to | `0.0.0.0` | +| `PORT` | `8080` | Port the server listens on | `8080` | diff --git a/app_go/docs/GO.md b/app_go/docs/GO.md new file mode 100644 index 0000000000..39785012a7 --- /dev/null +++ b/app_go/docs/GO.md @@ -0,0 +1,5 @@ +### Why Go? +- **Compiled binary**: produces a single executable (useful for multi-stage Docker builds). +- **Fast startup and low overhead**: good for microservices. +- **Standard library is enough**: `net/http` covers routing and HTTP server without external frameworks. +- **Great DevOps fit**: simple deployment, small runtime requirements. diff --git a/app_go/docs/LAB01.md b/app_go/docs/LAB01.md new file mode 100644 index 0000000000..52920f06bc --- /dev/null +++ b/app_go/docs/LAB01.md @@ -0,0 +1,217 @@ +# Lab 1 (Bonus) β€” DevOps Info Service in Go + +## 1. Language / Framework Selection + +### Choice +I implemented the bonus service in **Go** using the standard library **net/http** package. + +### Why Go? +- **Compiled binary**: produces a single executable (useful for multi-stage Docker builds). +- **Fast startup and low overhead**: good for microservices. +- **Standard library is enough**: `net/http` covers routing and HTTP server without external frameworks. +- **Great DevOps fit**: simple deployment, small runtime requirements. + +### Comparison with Alternatives + +| Criteria | Go (net/http) (chosen) | Rust | Java (Spring Boot) | C# (ASP.NET Core) | +|---------|--------------------------|------|---------------------|-------------------| +| Build artifact | Single binary | Single binary | JVM app + deps | .NET app + deps | +| Startup time | Fast | Fast | Usually slower | Medium | +| Runtime deps | None | None | JVM required | .NET runtime | +| HTTP stack | stdlib | frameworks (Axum/Actix) | Spring ecosystem | ASP.NET stack | +| Complexity | Low | Medium–high | Medium | Medium | +| Best fit for this lab | Excellent | Good | Overkill | Good | + +--- + +## 2. Best Practices Applied + +### 2.1 Clean Code Organization +- Clear data models (`ServiceInfo`, `Service`, `System`, `RuntimeInfo`, `RequestInfo`, `Endpoint`). +- Helper functions for concerns separation: + - `runtimeInfo()`, `requestInfo()`, `uptime()`, `isoUTCNow()`, `clientIP()`, `writeJSON()`. + +### 2.2 Configuration via Environment Variables +The service is configurable via environment variables: +- `HOST` (default `0.0.0.0`) +- `PORT` (default `8080`) +- `DEBUG` (default `false`) + +Implementation uses a simple helper: +```go +func getenv(key, def string) string { + v := os.Getenv(key) + if v == "" { + return def + } + return v +} +``` + +### 2.3 Logging Middleware +Request logging is implemented as middleware: +```go +func withLogging(logger *log.Logger) func(http.Handler) http.Handler { + return func(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + start := time.Now() + next.ServeHTTP(w, r) + logger.Printf("%s %s (%s) from %s in %s", + r.Method, r.URL.Path, r.Proto, r.RemoteAddr, time.Since(start)) + }) + } +} +``` + +### 2.4 Error Handling +#### 404 Not Found +Unknown endpoints return a consistent JSON error: +```json +{ + "error": "Not Found", + "message": "Endpoint does not exist" +} +``` +This is implemented via a wrapper that enforces valid paths: +```go +func withNotFound(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/" && r.URL.Path != "/health" { + writeJSON(w, http.StatusNotFound, ErrorResponse{ + Error: "Not Found", + Message: "Endpoint does not exist", + }) + return + } + next.ServeHTTP(w, r) + }) +} +``` +#### 500 Internal Server Error (panic recovery) +A recover middleware prevents crashes and returns a safe JSON response: +```go +func withRecover(logger *log.Logger) func(http.Handler) http.Handler { + return func(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + defer func() { + if rec := recover(); rec != nil { + logger.Printf("panic recovered: %v", rec) + writeJSON(w, http.StatusInternalServerError, ErrorResponse{ + Error: "Internal Server Error", + Message: "An unexpected error occurred", + }) + } + }() + next.ServeHTTP(w, r) + }) + } +} +``` +### 2.5 Production-Friendly HTTP Server Settings +The service uses `http.Server` with timeouts: +```go +srv := &http.Server{ + Addr: addr, + Handler: handler, + ReadHeaderTimeout: 5 * time.Second, +} +``` +## 3. API Documentation +### 3.1 GET / β€” Service and System Information +**Description**: Returns service metadata, system info, runtime info, request info, and available endpoints. + +**Request**: +```bash +curl -i http://127.0.0.1:8080/ +``` +**Response (200 OK) example**: +```json +{ + "service": { + "name": "devops-info-service", + "version": "1.0.0", + "description": "DevOps course info service", + "framework": "Go net/http" + }, + "system": { + "hostname": "DESKTOP-KUN1CI4", + "platform": "windows", + "platform_version": "unknown", + "architecture": "amd64", + "cpu_count": 8, + "go_version": "go1.25.6" + }, + "runtime": { + "uptime_seconds": 6, + "uptime_human": "0 hours, 0 minutes", + "current_time": "2026-01-25T17:17:32.248Z", + "timezone": "UTC" + }, + "request": { + "client_ip": "::1", + "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/144.0.0.0 Safari/537.36 Edg/144.0.0.0", + "method": "GET", + "path": "/" + }, + "endpoints": [ + { + "path": "/", + "method": "GET", + "description": "Service information" + }, + { + "path": "/health", + "method": "GET", + "description": "Health check" + } + ] +} +``` +### 3.2 GET /health β€” Health Check +**Description**: Description: Simple health endpoint used for monitoring and probes. + +**Request**: +```bash +curl -i http://127.0.0.1:8080/health +``` +**Response (200 OK) example**: +```json +{ + "status": "healthy", + "timestamp": "2026-01-25T17:19:02.582Z", + "uptime_seconds": 96 +} +``` +### 3.3 404 Behavior +**Request**: +```bash +curl -i http://127.0.0.1:8080/does-not-exist +``` +**Response (404 Not Found)**: +```json +{ + "error": "Not Found", + "message": "Endpoint does not exist" +} +``` + +## 4. Build & Run Instructions +### 4.1 Run locally (no build) +```bash +go run main.go +``` +### 4.2 Build binary +```bash +go build -o devops-info-service main.go +``` +Run: +```bash +./devops-info-service +``` +### 4.3 Environment variables examples +```bash +HOST=127.0.0.1 PORT=3000 ./devops-info-service +DEBUG=true PORT=8081 ./devops-info-service +``` +## 5. Challenges & Solutions +I don't know how `go` works. diff --git a/app_go/docs/LAB02.md b/app_go/docs/LAB02.md new file mode 100644 index 0000000000..ee98527851 --- /dev/null +++ b/app_go/docs/LAB02.md @@ -0,0 +1,215 @@ +# Multi-stage build strategy +## Stage 1 - Builder +**Purpose:** compile the Go application using a full Go toolchain image. + +**Key points:** +- Uses `golang:1.22-alpine` to keep the builder stage smaller than Debian-based images. +- Copies `go.mod` first and runs `go mod download` to maximize Docker layer caching. +- Builds a Linux binary with `CGO_ENABLED=0` (static binary), which allows a minimal runtime image. + +**Dockerfile snippet:** +```dockerfile +FROM golang:1.22-alpine AS builder +WORKDIR /app +COPY go.mod ./ +RUN go mod download +COPY . . +RUN CGO_ENABLED=0 go build -o myapp +``` + +## Stage 2 - Runtime +**Purpose:** run only the compiled binary in a minimal image with a non-root user. + +**Key points:** +- Uses `gcr.io/distroless/static-debian12:nonroot`. +- Distroless images contain only what is required to run the app (no package manager, no shell), reducing image size and attack surface. +- Runs as a **non-root** user (provided by the `:nonroot` tag). + +**Dockerfile snippet:** +```dockerfile +FROM gcr.io/distroless/static-debian12:nonroot +WORKDIR /app +COPY --from=builder /app/myapp . +EXPOSE 8080 +CMD ["./myapp"] +``` + +# Size comparison with analysis (builder vs final image) +## Builder +```bash +docker images newspec/app_go:builder +REPOSITORY TAG IMAGE ID CREATED SIZE +newspec/app_go builder 21b01f8c6103 37 minutes ago 305MB +``` +## Final +```bash +docker images newspec/app_go:1.0 +REPOSITORY TAG IMAGE ID CREATED SIZE +newspec/app_go 1.0 a944205e6030 59 minutes ago 9.32MB +``` +## Analysis +The builder image contains the Go toolchain and build dependencies, so it is significantly larger. +The final image contains only the static binary, which is much smaller and safer. + +# Why multi-stage builds matter for compiled languages + +Compiled languages (like Go, Rust, Java with native images, etc.) typically require a **heavy build environment**: compilers, linkers, SDKs, package managers, and temporary build artifacts. If you ship that same environment as your runtime container, the final image becomes unnecessarily large and less secure. + +Multi-stage builds solve this by separating concerns: + +### 1) Smaller final images (faster pull & deploy) +- The **builder stage** includes the full toolchain (large). +- The **runtime stage** contains only the compiled output (usually just a single binary). +This dramatically reduces image size, which improves: +- CI/CD speed (less time to push/pull) +- Startup speed (faster distribution in clusters) +- Bandwidth/storage usage + +### 2) Better security (smaller attack surface) +The runtime image no longer contains: +- compilers (go, gcc, build tools) +- package managers +- shells and utilities (especially with distroless/scratch) + +Fewer components, so fewer potential vulnerabilities (CVEs) and fewer tools available to an attacker if the container is compromised. + +### 3) Cleaner separation of build vs runtime +Multi-stage builds enforce a clear boundary: +- build dependencies exist only where needed (builder stage) +- runtime image stays minimal and focused on execution + +This makes the container easier to reason about and maintain. + +### 4) Reproducible and cache-friendly builds +When the Dockerfile copies dependency descriptors first (e.g., `go.mod`/`go.sum`) and downloads dependencies before copying source code: +- Docker can reuse cached layers if dependencies are unchanged +- rebuilds after code changes are significantly faster + +### 5) Enables ultra-minimal runtime images +Compiled apps can often run in very small base images: +- `distroless` (secure and minimal) +- `scratch` (almost empty) + +This is usually impossible for interpreted languages without bundling an interpreter runtime. + +**Summary:** For compiled languages, multi-stage builds provide the best of both worlds β€” a full build environment when you need it, and a minimal secure runtime image when you deploy. + +# Terminal output showing build process +## Builder +```bash +docker build --target builder -t newspec/app_go:builder . +[+] Building 2.4s (12/12) FINISHED docker:desktop-linux + => [internal] load build definition from Dockerfile 0.0s + => => transferring dockerfile: 349B 0.0s + => [internal] load metadata for docker.io/library/golang:1.22-alpine 2.1s + => [auth] library/golang:pull token for registry-1.docker.io 0.0s + => [internal] load .dockerignore 0.1s + => => transferring context: 105B 0.0s + => [builder 1/6] FROM docker.io/library/golang:1.22-alpine@sha256:1699c10032ca2582ec89a24a1312d986a3f094aed3d5c1147b19880afe40e052 0.0s + => [internal] load build context 0.0s + => => transferring context: 146B 0.0s + => CACHED [builder 2/6] WORKDIR /app 0.0s + => CACHED [builder 3/6] COPY go.mod ./ 0.0s + => CACHED [builder 4/6] RUN go mod download 0.0s + => CACHED [builder 5/6] COPY . . 0.0s + => CACHED [builder 6/6] RUN CGO_ENABLED=0 go build -o myapp 0.0s + => exporting to image 0.0s + => => exporting layers 0.0s + => => writing image sha256:21b01f8c61038149b9130afe7881765d625b2eb6622b6b46f42682d26b10ae2b 0.0s + => => naming to docker.io/newspec/app_go:builder 0.0s + +View build details: docker-desktop://dashboard/build/desktop-linux/desktop-linux/wvw3g1yzoqu1uput2mz8me7zx +``` + +## Final +```bash +docker build -t newspec/app_go:1.0 . +[+] Building 1.5s (15/15) FINISHED docker:desktop-linux + => [internal] load build definition from Dockerfile 0.0s + => => transferring dockerfile: 349B 0.0s + => [internal] load metadata for gcr.io/distroless/static-debian12:nonroot 1.0s + => [internal] load metadata for docker.io/library/golang:1.22-alpine 0.7s + => [internal] load .dockerignore 0.0s + => => transferring context: 105B 0.0s + => [builder 1/6] FROM docker.io/library/golang:1.22-alpine@sha256:1699c10032ca2582ec89a24a1312d986a3f094aed3d5c1147b19880afe40e052 0.0s + => [stage-1 1/3] FROM gcr.io/distroless/static-debian12:nonroot@sha256:cba10d7abd3e203428e86f5b2d7fd5eb7d8987c387864ae4996cf97191b33764 0.0s + => [internal] load build context 0.0s + => => transferring context: 146B 0.0s + => CACHED [builder 2/6] WORKDIR /app 0.0s + => CACHED [builder 3/6] COPY go.mod ./ 0.0s + => CACHED [builder 4/6] RUN go mod download 0.0s + => CACHED [builder 5/6] COPY . . 0.0s + => CACHED [builder 6/6] RUN CGO_ENABLED=0 go build -o myapp 0.0s + => CACHED [stage-1 2/3] WORKDIR /app 0.0s + => CACHED [stage-1 3/3] COPY --from=builder /app/myapp . 0.0s + => exporting to image 0.1s + => => exporting layers 0.0s + => => writing image sha256:c9ff1572d8a13240f00ef7d66683264e0fbf4fa77c12790dc3f3428972819321 0.0s + => => naming to docker.io/newspec/app_go:1.0 0.0s + +View build details: docker-desktop://dashboard/build/desktop-linux/desktop-linux/nvdyhylzo1hzpemy23lt42ll1 +``` +# Technical explanation of each stage's purpose +### Stage 1 β€” Builder (Compile Environment) +**Goal:** Produce a Linux executable from Go source code in a controlled build environment. + +**Why this stage exists:** +- Go compilation requires the Go toolchain (compiler, linker) which is large and should not be shipped in the final runtime image. +- The builder image provides everything needed to compile the application. + +**What happens technically:** +1. **Set working directory** + - `WORKDIR /app` defines where source code and build steps run inside the container. + +2. **Copy dependency definition first** + - `COPY go.mod ./` is done before copying the whole source tree. + - This allows Docker to cache the dependency download layer. + - Even if code changes, dependencies may not, so rebuilds are faster. + +3. **Download modules** + - `RUN go mod download` fetches required modules. + - In this project there are no external module dependencies, so Go prints: + `go: no module dependencies to download` + - The step is still good practice and keeps the Dockerfile consistent for future changes. + +4. **Copy application source code** + - `COPY . .` brings in the Go source files. + - This layer changes most often, so it comes after dependency caching steps. + +5. **Compile a static binary** + - `RUN CGO_ENABLED=0 go build -o myapp` + - `CGO_ENABLED=0` disables C bindings so the binary is statically linked. + - A static binary does not require libc or other runtime shared libraries, enabling minimal runtime images. + +**Output of the stage:** a compiled executable (`/app/myapp`). + +--- + +### Stage 2 β€” Runtime (Execution Environment) +**Goal:** Run only the compiled binary in a minimal and secure container image. + +**Why this stage exists:** +- The runtime stage should not contain compilers, source code, or build tools. +- A smaller runtime image reduces attack surface and improves deployment speed. + +**What happens technically:** +1. **Choose a minimal base image** + - `FROM gcr.io/distroless/static-debian12:nonroot` + - Distroless images contain only the minimum required runtime files. + - The `:nonroot` variant runs as a non-root user by default. + +2. **Set working directory** + - `WORKDIR /app` provides a predictable location for the binary. + +3. **Copy only the build artifact** + - `COPY --from=builder /app/myapp .` + - This copies only the compiled binary from the builder stage. + - No source code, no Go toolchain, no dependency caches are included. + +4. **Run the application** + - `CMD ["./myapp"]` starts the service. + - The application reads `HOST` and `PORT` environment variables: + - defaults: `HOST=0.0.0.0`, `PORT=8080` + - When running the container, port mapping must match the internal listening port (e.g., `-p 8000:8080`). + +**Output of the stage:** a minimal runtime container that executes the Go binary as a non-root user. \ No newline at end of file diff --git a/app_go/docs/LAB03.md b/app_go/docs/LAB03.md new file mode 100644 index 0000000000..e039089a71 --- /dev/null +++ b/app_go/docs/LAB03.md @@ -0,0 +1,86 @@ +# LAB03 (Go) β€” Bonus: Multi-App CI + Path Filters + Coverage + +## 1) Second workflow implementation (Go CI) + language-specific best practices + +A separate workflow file is added for the Go application: + +- `.github/workflows/go-ci.yml` + +It implements Go-specific CI best practices: + +- **Setup Go toolchain** via `actions/setup-go` (Go 1.22+) +- **Formatting check**: `gofmt -l .` must return empty output +- **Linting**: `golangci-lint` (industry-standard Go linter aggregator) +- **Unit tests**: `go test ./...` +- **Coverage generation**: `go test -coverprofile=coverage.out ./...` +- **Docker build/push**: multi-stage Docker build using the existing `app_go/Dockerfile` (builder stage + distroless runtime) + +Docker image tagging follows the same CalVer strategy as Python: + +- `YYYY.MM.DD` (CalVer) +- `${GITHUB_SHA}` (commit SHA) +- `latest` + +## 2) Path filter configuration + testing proof + +The Go workflow is triggered only when Go-related files change: + +- `app_go/**` +- `.github/workflows/go-ci.yml` + +This prevents unnecessary CI runs for unrelated parts of the monorepo. + +### Proof (selective triggering) + +Provide evidence with 2 small commits: + +1) **Change only Go files** + - https://github.com/newspec/DevOps-Core-Course/actions/runs/21837847722 + +2) **Change only Python files** + - https://github.com/newspec/DevOps-Core-Course/actions/runs/21838134121 + +## 3) Benefits analysis β€” why path filters matter in monorepos + +Path filters are important in monorepos because they: + +- **Save CI time and compute**: no need to run Go CI when only Python changes (and vice versa) +- **Reduce noise in PR checks**: fewer irrelevant checks, faster feedback for reviewers +- **Improve developer experience**: faster iteration and fewer β€œunrelated failures” +- **Scale better** as more apps/labs are added to the same repository + +## 4) Example showing workflows running independently + +Example scenario: + +- Commit A modifies only `app_go/**` β†’ only Go CI runs +- Commit B modifies only `app_python/**` β†’ only Python CI runs +- Commit C modifies both `app_go/**` and `app_python/**` β†’ both workflows run in parallel + +![screenshots/proof_of_path_validations.png](screenshots/proof_of_path_validations.png) + +## 5) Terminal output / Actions evidence (selective triggering) + +See links and screenshot above. + +## 6) Coverage integration (dashboard link / screenshot) +![screenshots/coverage.png](screenshots/coverage.png) + +## 7) Coverage analysis (current percentage, covered/not covered, threshold) + +### Current coverage + +Current Go coverage: 44% +Current Python coverage 99% + +### What is covered (Go) +- `GET /` handler (`mainHandler`) returns status `200` and contains required top-level JSON keys: + `service`, `system`, `runtime`, `request`, `endpoints` +- `GET /health` handler (`healthHandler`) returns status `200` and contains required keys: + `status`, `timestamp`, `uptime_seconds` + +### What is not covered (Go) +- Middleware behavior (`withRecover`, `withLogging`, `withNotFound`) +- Negative/error scenarios (e.g., unknown path via middleware, panic recovery) +- Strict validation of dynamic fields (timestamps formatting beyond basic checks) + diff --git a/app_go/docs/screenshots/01-main-endpoint.png b/app_go/docs/screenshots/01-main-endpoint.png new file mode 100644 index 0000000000..c0f1b2ed5c Binary files /dev/null and b/app_go/docs/screenshots/01-main-endpoint.png differ diff --git a/app_go/docs/screenshots/02-health-check.png b/app_go/docs/screenshots/02-health-check.png new file mode 100644 index 0000000000..aed7a37918 Binary files /dev/null and b/app_go/docs/screenshots/02-health-check.png differ diff --git a/app_go/docs/screenshots/03-formatted-output.png b/app_go/docs/screenshots/03-formatted-output.png new file mode 100644 index 0000000000..ec011296fc Binary files /dev/null and b/app_go/docs/screenshots/03-formatted-output.png differ diff --git a/app_go/docs/screenshots/coverage.png b/app_go/docs/screenshots/coverage.png new file mode 100644 index 0000000000..1c9c2434ef Binary files /dev/null and b/app_go/docs/screenshots/coverage.png differ diff --git a/app_go/docs/screenshots/proof_of_path_validations.png b/app_go/docs/screenshots/proof_of_path_validations.png new file mode 100644 index 0000000000..00df9e7997 Binary files /dev/null and b/app_go/docs/screenshots/proof_of_path_validations.png differ diff --git a/app_go/go.mod b/app_go/go.mod new file mode 100644 index 0000000000..43fa976d01 --- /dev/null +++ b/app_go/go.mod @@ -0,0 +1,3 @@ +module devops-info-service + +go 1.22 \ No newline at end of file diff --git a/app_go/main.go b/app_go/main.go new file mode 100644 index 0000000000..1e5f3f1abe --- /dev/null +++ b/app_go/main.go @@ -0,0 +1,277 @@ +package main + +import ( + "encoding/json" + "fmt" + "log" + "net" + "net/http" + "os" + "runtime" + "strings" + "time" +) + +type ServiceInfo struct { + Service Service `json:"service"` + System System `json:"system"` + Runtime RuntimeInfo `json:"runtime"` + Request RequestInfo `json:"request"` + Endpoints []Endpoint `json:"endpoints"` +} + +type Service struct { + Name string `json:"name"` + Version string `json:"version"` + Description string `json:"description"` + Framework string `json:"framework"` +} + +type System struct { + Hostname string `json:"hostname"` + Platform string `json:"platform"` + PlatformVersion string `json:"platform_version"` + Architecture string `json:"architecture"` + CPUCount int `json:"cpu_count"` + GoVersion string `json:"go_version"` +} + +type RuntimeInfo struct { + UptimeSeconds int `json:"uptime_seconds"` + UptimeHuman string `json:"uptime_human"` + CurrentTime string `json:"current_time"` + Timezone string `json:"timezone"` +} + +type RequestInfo struct { + ClientIP string `json:"client_ip"` + UserAgent string `json:"user_agent"` + Method string `json:"method"` + Path string `json:"path"` +} + +type Endpoint struct { + Path string `json:"path"` + Method string `json:"method"` + Description string `json:"description"` +} + +type ErrorResponse struct { + Error string `json:"error"` + Message string `json:"message"` +} + +var startTime = time.Now().UTC() + +func main() { + host := getenv("HOST", "0.0.0.0") + port := getenv("PORT", "8080") + debug := strings.ToLower(getenv("DEBUG", "false")) == "true" + + logger := log.New(os.Stdout, "", log.LstdFlags) + if debug { + logger.SetFlags(log.LstdFlags | log.Lshortfile) + } + + mux := http.NewServeMux() + + // endpoints + mux.HandleFunc("/", mainHandler) + mux.HandleFunc("/health", healthHandler) + + // wrap with middleware: recover + logging + 404 + handler := withRecover(logger)(withLogging(logger)(withNotFound(mux))) + + addr := fmt.Sprintf("%s:%s", host, port) + logger.Printf("Application starting on http://%s\n", addr) + + // http.Server allows timeouts (good practice) + srv := &http.Server{ + Addr: addr, + Handler: handler, + ReadHeaderTimeout: 5 * time.Second, + } + + if err := srv.ListenAndServe(); err != nil { + logger.Fatalf("server error: %v", err) + } +} + +func mainHandler(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/" { + // will be caught by notFound wrapper, but this is extra safety + writeJSON(w, http.StatusNotFound, ErrorResponse{ + Error: "Not Found", + Message: "Endpoint does not exist", + }) + return + } + + info := ServiceInfo{ + Service: Service{ + Name: "devops-info-service", + Version: "1.0.0", + Description: "DevOps course info service", + Framework: "Go net/http", + }, + System: System{ + Hostname: hostname(), + Platform: runtime.GOOS, + PlatformVersion: platformVersion(), + Architecture: runtime.GOARCH, + CPUCount: runtime.NumCPU(), + GoVersion: runtime.Version(), + }, + Runtime: runtimeInfo(), + Request: requestInfo(r), + Endpoints: []Endpoint{ + {Path: "/", Method: "GET", Description: "Service information"}, + {Path: "/health", Method: "GET", Description: "Health check"}, + }, + } + + writeJSON(w, http.StatusOK, info) +} + +func healthHandler(w http.ResponseWriter, r *http.Request) { + uptimeSeconds, _ := uptime() + resp := map[string]any{ + "status": "healthy", + "timestamp": isoUTCNow(), + "uptime_seconds": uptimeSeconds, + } + writeJSON(w, http.StatusOK, resp) +} + +func runtimeInfo() RuntimeInfo { + secs, human := uptime() + return RuntimeInfo{ + UptimeSeconds: secs, + UptimeHuman: human, + CurrentTime: isoUTCNow(), + Timezone: "UTC", + } +} + +func requestInfo(r *http.Request) RequestInfo { + ip := clientIP(r) + ua := r.Header.Get("User-Agent") + return RequestInfo{ + ClientIP: ip, + UserAgent: ua, + Method: r.Method, + Path: r.URL.Path, + } +} + +func uptime() (int, string) { + delta := time.Since(startTime) + seconds := int(delta.Seconds()) + hours := seconds / 3600 + minutes := (seconds % 3600) / 60 + return seconds, fmt.Sprintf("%d hours, %d minutes", hours, minutes) +} + +func isoUTCNow() string { + // "2026-01-07T14:30:00.000Z" + return time.Now().UTC().Format("2006-01-02T15:04:05.000Z") +} + +func hostname() string { + h, err := os.Hostname() + if err != nil { + return "unknown" + } + return h +} + +func platformVersion() string { + // Best effort for Linux: /etc/os-release PRETTY_NAME (e.g., "Ubuntu 24.04.1 LTS") + if runtime.GOOS != "linux" { + return "unknown" + } + data, err := os.ReadFile("/etc/os-release") + if err != nil { + return "unknown" + } + lines := strings.Split(string(data), "\n") + for _, line := range lines { + if strings.HasPrefix(line, "PRETTY_NAME=") { + val := strings.TrimPrefix(line, "PRETTY_NAME=") + val = strings.Trim(val, `"`) + if val != "" { + return val + } + } + } + return "unknown" +} + +func clientIP(r *http.Request) string { + // If behind proxy, you might consider X-Forwarded-For, but for lab keep it simple. + host, _, err := net.SplitHostPort(r.RemoteAddr) + if err == nil && host != "" { + return host + } + // fallback: may already be just an IP + return r.RemoteAddr +} + +func writeJSON(w http.ResponseWriter, statusCode int, payload any) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(statusCode) + _ = json.NewEncoder(w).Encode(payload) +} + +func getenv(key, def string) string { + v := os.Getenv(key) + if v == "" { + return def + } + return v +} + +/* ---------------- Middleware (Best Practices) ---------------- */ + +func withLogging(logger *log.Logger) func(http.Handler) http.Handler { + return func(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + start := time.Now() + next.ServeHTTP(w, r) + logger.Printf("%s %s (%s) from %s in %s", + r.Method, r.URL.Path, r.Proto, r.RemoteAddr, time.Since(start)) + }) + } +} + +func withRecover(logger *log.Logger) func(http.Handler) http.Handler { + return func(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + defer func() { + if rec := recover(); rec != nil { + logger.Printf("panic recovered: %v", rec) + writeJSON(w, http.StatusInternalServerError, ErrorResponse{ + Error: "Internal Server Error", + Message: "An unexpected error occurred", + }) + } + }() + next.ServeHTTP(w, r) + }) + } +} + +func withNotFound(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + // Use ServeMux; if it doesn't match, it still calls handler with pattern "/" + // So we enforce our own 404 for unknown endpoints. + if r.URL.Path != "/" && r.URL.Path != "/health" { + writeJSON(w, http.StatusNotFound, ErrorResponse{ + Error: "Not Found", + Message: "Endpoint does not exist", + }) + return + } + next.ServeHTTP(w, r) + }) +} diff --git a/app_go/main_test.go b/app_go/main_test.go new file mode 100644 index 0000000000..a273d1f240 --- /dev/null +++ b/app_go/main_test.go @@ -0,0 +1,56 @@ +package main + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "testing" +) + +func TestRootOK(t *testing.T) { + req := httptest.NewRequest(http.MethodGet, "/", nil) + rr := httptest.NewRecorder() + + mainHandler(rr, req) + + if rr.Code != http.StatusOK { + t.Fatalf("expected 200, got %d", rr.Code) + } + + var data map[string]any + if err := json.Unmarshal(rr.Body.Bytes(), &data); err != nil { + t.Fatalf("invalid json: %v", err) + } + + // top-level keys + for _, k := range []string{"service", "system", "runtime", "request", "endpoints"} { + if _, ok := data[k]; !ok { + t.Fatalf("missing key: %s", k) + } + } +} + +func TestHealthOK(t *testing.T) { + req := httptest.NewRequest(http.MethodGet, "/health", nil) + rr := httptest.NewRecorder() + + healthHandler(rr, req) + + if rr.Code != http.StatusOK { + t.Fatalf("expected 200, got %d", rr.Code) + } + + var data map[string]any + if err := json.Unmarshal(rr.Body.Bytes(), &data); err != nil { + t.Fatalf("invalid json: %v", err) + } + + for _, k := range []string{"status", "timestamp", "uptime_seconds"} { + if _, ok := data[k]; !ok { + t.Fatalf("missing key: %s", k) + } + } + if data["status"] != "healthy" { + t.Fatalf("expected status healthy, got %v", data["status"]) + } +} diff --git a/app_python/.dockerignore b/app_python/.dockerignore new file mode 100644 index 0000000000..44fa25304b --- /dev/null +++ b/app_python/.dockerignore @@ -0,0 +1,22 @@ +# πŸ™ Version control +.git +.gitignore + +# 🐍 Python +__pycache__ +*.pyc +*.pyo +venv/ +.venv/ + +# πŸ” Secrets (NEVER include!) +.env +*.pem +secrets/ + +# πŸ“ Documentation +*.md +docs/ + +# πŸ§ͺ Tests (if not needed in container) +tests/ \ No newline at end of file diff --git a/app_python/.gitignore b/app_python/.gitignore new file mode 100644 index 0000000000..4de420a8f7 --- /dev/null +++ b/app_python/.gitignore @@ -0,0 +1,12 @@ +# Python +__pycache__/ +*.py[cod] +venv/ +*.log + +# IDE +.vscode/ +.idea/ + +# OS +.DS_Store \ No newline at end of file diff --git a/app_python/Dockerfile b/app_python/Dockerfile new file mode 100644 index 0000000000..0364cfd9a7 --- /dev/null +++ b/app_python/Dockerfile @@ -0,0 +1,14 @@ +FROM python:3.12-slim + +RUN useradd --create-home --shell /bin/bash appuser + +WORKDIR /app +COPY requirements.txt . +RUN pip install --no-cache-dir -r requirements.txt +COPY app.py . + +EXPOSE 8000 + +USER appuser + +CMD ["python", "app.py"] \ No newline at end of file diff --git a/app_python/README.md b/app_python/README.md new file mode 100644 index 0000000000..170f1cac89 --- /dev/null +++ b/app_python/README.md @@ -0,0 +1,76 @@ +[![Python CI](https://github.com/newspec/DevOps-Core-Course/actions/workflows/python-ci.yml/badge.svg?branch=lab03)](https://github.com/newspec/DevOps-Core-Course/actions/workflows/python-ci.yml?query=branch%3Alab03) +[![Coverage](https://codecov.io/gh/newspec/DevOps-Core-Course/branch/lab03/graph/badge.svg?flag=python)](https://codecov.io/gh/newspec/DevOps-Core-Course/branch/lab03?flag=python) + + +# devops-info-service + +## Overview +`devops-info-service` is a lightweight HTTP service built with **FastAPI** that returns comprehensive runtime and system information. It exposes: +- service metadata (name, version, description, framework), +- system details (hostname, OS/platform, architecture, CPU count, Python version), +- runtime data (uptime, current UTC time), +- request details (client IP, user-agent, method, path), +- a list of available endpoints. + +## Prerequisites +- **Python:** 3.10+ (recommended 3.11+) +- **Dependencies:** listed in `requirements.txt` + +## Installation + +``` +python -m venv venv +source venv/bin/activate +pip install -r requirements.txt +``` + +## Running the Application +``` +python app.py +# Or with custom config +PORT=8080 python app.py +``` + +## API Endpoints +- `GET /` - Service and system information +- `GET /health` - Health check + +## Configuration + +The application is configured using environment variables. + +| Variable | Default | Description | Example | +|---------|---------|-------------|-------------| +| `HOST` | `0.0.0.0` | Host interface to bind the server to | `127.0.0.1` | +| `PORT` | `8000` | Port the server listens on | `8080` | + +# Docker + +## Building the image locally +Command pattern: +```bash +docker build -t : +``` + +## Running a container +Command pattern: +```bash +docker run --rm -p : : +``` + +## Pulling from Docker Hub +Command pattern: +```bash +docker pull /: +``` +Then run: +```bash +docker run --rm -p : /: +``` + +# Testing +To run test locally use command: +```bash +pytest +``` + diff --git a/app_python/app.py b/app_python/app.py new file mode 100644 index 0000000000..9e69e51883 --- /dev/null +++ b/app_python/app.py @@ -0,0 +1,260 @@ +""" +DevOps Info Service +Main application module +""" +import json +import logging +import os +import platform +import socket +import sys +from datetime import datetime, timezone + +import uvicorn +from fastapi import FastAPI, Request, status +from fastapi.responses import JSONResponse +from pythonjsonlogger import jsonlogger +from starlette.exceptions import HTTPException + +app = FastAPI() + +# Configuration +HOST = os.getenv("HOST", "0.0.0.0") +PORT = int(os.getenv("PORT", 8000)) +DEBUG = os.getenv("DEBUG", "False").lower() == "true" + +# JSON Logging Configuration +class CustomJsonFormatter(jsonlogger.JsonFormatter): + """Custom JSON formatter with additional fields.""" + + def add_fields(self, log_record, record, message_dict): + super(CustomJsonFormatter, self).add_fields(log_record, record, message_dict) + log_record['timestamp'] = datetime.now(timezone.utc).isoformat() + log_record['level'] = record.levelname + log_record['logger'] = record.name + log_record['service'] = 'devops-python' + +# Setup JSON logging +logHandler = logging.StreamHandler(sys.stdout) +formatter = CustomJsonFormatter('%(timestamp)s %(level)s %(name)s %(message)s') +logHandler.setFormatter(formatter) +logger = logging.getLogger() +logger.addHandler(logHandler) +logger.setLevel(logging.INFO) + +# Get module logger +logger = logging.getLogger(__name__) + +# Application start time +start_time = datetime.now() + + +@app.middleware("http") +async def log_requests(request: Request, call_next): + """Middleware to log all HTTP requests and responses.""" + # Log incoming request + logger.info( + "Incoming request", + extra={ + "method": request.method, + "path": request.url.path, + "client_ip": request.client.host if request.client else "unknown", + "user_agent": request.headers.get("user-agent", "unknown"), + } + ) + + # Process request + try: + response = await call_next(request) + + # Log response + logger.info( + "Request completed", + extra={ + "method": request.method, + "path": request.url.path, + "status_code": response.status_code, + "client_ip": request.client.host if request.client else "unknown", + } + ) + + return response + except Exception as e: + logger.error( + "Request failed", + extra={ + "method": request.method, + "path": request.url.path, + "error": str(e), + "client_ip": request.client.host if request.client else "unknown", + }, + exc_info=True + ) + raise + + +def get_service_info(): + """Get information about service.""" + logger.debug('Getting info about the service.') + return { + "name": "devops-info-service", + "version": "1.0.0", + "description": "DevOps course info service", + "framework": "FastAPI", + } + + +def get_system_info(): + """Get information about system.""" + logger.debug('Getting info about the system.') + return { + "hostname": socket.gethostname(), + "platform": platform.system(), + "platform_version": platform.version(), + "architecture": platform.machine(), + "cpu_count": os.cpu_count(), + "python_version": platform.python_version(), + } + + +def get_uptime(): + """Get uptime.""" + logger.debug('Getting uptime.') + delta = datetime.now() - start_time + seconds = int(delta.total_seconds()) + hours = seconds // 3600 + minutes = (seconds % 3600) // 60 + return {"seconds": seconds, "human": f"{hours} hours, {minutes} minutes"} + + +def get_runtime_info(): + """Get information about runtime.""" + logger.debug('Getting runtime info.') + uptime = get_uptime() + uptime_seconds, uptime_human = uptime["seconds"], uptime["human"] + current_time = datetime.now(timezone.utc) + + return { + "uptime_seconds": uptime_seconds, + "uptime_human": uptime_human, + "current_time": current_time, + "timezone": "UTC", + } + + +def get_request_info(request: Request): + """Get information about request.""" + logger.debug('Getting info about request.') + return { + "client_ip": request.client.host, + "user_agent": request.headers.get("user-agent"), + "method": request.method, + "path": request.url.path, + } + + +def get_endpoints(): + """Get all existing ednpoints.""" + logger.debug('Getting list of all endpoints.') + return [ + {"path": "/", "method": "GET", "description": "Service information"}, + {"path": "/health", "method": "GET", "description": "Health check"}, + ] + + +@app.get("/", status_code=status.HTTP_200_OK) +async def root(request: Request): + """Main endpoint - service and system information.""" + logger.info("Processing root endpoint request") + return { + "service": get_service_info(), + "system": get_system_info(), + "runtime": get_runtime_info(), + "request": get_request_info(request), + "endpoints": get_endpoints(), + } + + +@app.get("/health", status_code=status.HTTP_200_OK) +async def health(request: Request): + """Endpoint to check health.""" + logger.info("Health check requested") + return { + "status": "healthy", + "timestamp": datetime.now(timezone.utc), + "uptime_seconds": get_uptime()["seconds"], + } + + +@app.exception_handler(HTTPException) +async def http_exception_handler(request: Request, exc: HTTPException): + """Exception 404 (Not found) that endpoint does not exists.""" + logger.error( + "HTTP exception occurred", + extra={ + "status_code": exc.status_code, + "path": request.url.path, + "detail": exc.detail, + } + ) + + if exc.status_code == 404: + return JSONResponse( + status_code=404, + content={ + "error": "Not Found", + "message": "Endpoint does not exist", + }, + ) + return JSONResponse( + status_code=exc.status_code, + content={ + "error": "HTTP Error", + "message": exc.detail if exc.detail else "Request failed", + }, + ) + + +@app.exception_handler(Exception) +async def unhandled_exception_handler(request: Request, exc: Exception): + """Exception 500 (Internal Server Error) - For any unhandled errors.""" + logger.error( + "Unhandled exception occurred", + extra={ + "path": request.url.path, + "error_type": type(exc).__name__, + "error_message": str(exc), + }, + exc_info=True + ) + + return JSONResponse( + status_code=500, + content={ + "error": "Internal Server Error", + "message": "An unexpected error occurred", + }, + ) + + +if __name__ == "__main__": + # The entry point + logger.info( + "Application starting", + extra={ + "host": HOST, + "port": PORT, + "debug": DEBUG, + "python_version": platform.python_version(), + } + ) + + # Disable uvicorn access logs to keep only JSON logs + uvicorn.run( + "app:app", + host=HOST, + port=PORT, + reload=True, + log_config=None, # Disable default logging + access_log=False # Disable access logs + ) diff --git a/app_python/docs/LAB01.md b/app_python/docs/LAB01.md new file mode 100644 index 0000000000..1d43f6863d --- /dev/null +++ b/app_python/docs/LAB01.md @@ -0,0 +1,341 @@ +# 1. Framework selection. +## Choice +I have chose FastAPI because I have had an experience with it and have no experience with other frameworks. + +## Comparison with Alternatives + +| Criteria | FastAPI (chosen) | Flask | Django (DRF) | +|---------|-----------------|-------|--------------| +| Primary use | APIs / microservices | Lightweight web apps & APIs | Full-stack apps & large APIs | +| Performance model | ASGI (async-ready) | WSGI (sync by default) | WSGI/ASGI (heavier stack) | +| Built-in API docs | Yes (Swagger/OpenAPI) | No (manual/add-ons) | Yes (via DRF) | +| Validation / typing | Strong (type hints + Pydantic) | Manual or extensions | Strong (serializers) | +| Boilerplate | Low | Very low | Higher | +| Learning curve | Low–medium | Low | Medium–high | +| Best fit for this lab | Excellent | Good | Overkill | + +--- + +# 2. Best Practices Applied +## Clean Code Organization + +### 1) Clear Function Names +The code uses descriptive, intention-revealing function names that clearly communicate what each block returns: + +```python +def get_service_info(): + """Get information about service.""" + ... + +def get_system_info(): + """Get information about system.""" + ... + +def get_runtime_info(): + """Get information about runtime.""" + ... + +def get_request_info(request: Request): + """Get information about request.""" + ... +``` +**Why it matters**: Clear naming improves readability, reduces the need for extra comments, and makes the code easier to maintain and extend. + +### 2) Proper imports grouping +Imports are organized by category (standard library first, then third-party libraries), which is the common Python convention: +```python +import logging +import os +import platform +import socket +from datetime import datetime, timezone + +import uvicorn +from fastapi import FastAPI, Request, status +from fastapi.responses import JSONResponse +from starlette.exceptions import HTTPException +``` +**Why it matters**: Grouped imports make dependencies easier to understand at a glance, help keep the file structured, and align with typical linting rules. + +### 3) Comments only where needed +Instead of excessive inline comments, the code relies on clear names and short docstrings: +```python +""" +DevOps Info Service +Main application module +""" + +def get_uptime(): + """Get uptime.""" + ... +``` +**Why it matters**: Too many comments can become outdated. Minimal documentation plus clean naming keeps the codebase readable and accurate. + +### 4) Follow PEP 8 +The implementation follows common PEP 8 practices: +- consistent indentation and spacing, +- snake_case for variables and function names, +- configuration/constants placed near the top of the module (HOST, PORT, DEBUG), +- readable multi-line formatting for long calls: +```python +""" +logging.basicConfig( + level=logging.INFO, + format="%(asctime)s - %(name)s - %(levelname)s - %(message)s" +) +``` +**Why it matters**: PEP 8 improves consistency, supports teamwork, and makes the code compatible with linters/formatters such as `flake8`, `ruff`, and `black`. + +## Error Handling +The service implements centralized error handling using FastAPI/Starlette exception handlers. This ensures that errors are returned in a consistent JSON format and that clients receive meaningful messages instead of raw stack traces. + +### HTTP errors (e.g., 404 Not Found) +A dedicated handler processes HTTP-related exceptions and customizes the response for missing endpoints. + +```python +from starlette.exceptions import HTTPException + +@app.exception_handler(HTTPException) +async def http_exception_handler(request: Request, exc: HTTPException): + if exc.status_code == 404: + return JSONResponse( + status_code=404, + content={ + "error": "Not Found", + "message": "Endpoint does not exist", + }, + ) + + return JSONResponse( + status_code=exc.status_code, + content={ + "error": "HTTP Error", + "message": exc.detail if exc.detail else "Request failed", + }, + ) +``` +**Why it matters**: +- Provides a clear and user-friendly message for invalid routes. +- Keeps error responses consistent across the API. +- Avoids exposing internal implementation details to the client. + +### Unhandled exceptions (500 Internal Server Error) +A global handler catches any unexpected exceptions and returns a safe, standardized response. + +```python +@app.exception_handler(Exception) +async def unhandled_exception_handler(request: Request, exc: Exception): + return JSONResponse( + status_code=500, + content={ + "error": "Internal Server Error", + "message": "An unexpected error occurred", + }, + ) +``` +**Why it matters**: +- Prevents server crashes from unhandled errors. +- Ensures clients always receive valid JSON (important for automation/scripts). +- Helps keep production behavior predictable while preserving the option to log the exception internally. + +## 3. Logging +The service includes basic logging configuration to improve observability and simplify debugging. Logs are useful both during development (troubleshooting requests and behavior) and in production (monitoring, incident investigation). + +### Logging setup +A global logging configuration is defined at startup with a consistent log format: +```python +import logging + +logging.basicConfig( + level=logging.INFO, + format="%(asctime)s - %(name)s - %(levelname)s - %(message)s" +) +logger = logging.getLogger(__name__) +``` +**Why it matters**: +- Provides timestamps and log levels for easier troubleshooting. +- A consistent format makes logs easier to parse in log aggregators (e.g., ELK, Loki). +- Centralized config avoids inconsistent logging across modules. + +### Startup logging +The application logs an informational message when it starts: +```python +if __name__ == "__main__": + logger.info("Application starting...") + uvicorn.run("app:app", host=HOST, port=PORT, reload=True) +``` +**Why it matters**: +- Confirms that the service started successfully. +- Helps identify restarts and uptime issues. + +### Request logging (debug level) +Each endpoint logs basic request information (method and path): +```python +@app.get("/", status_code=status.HTTP_200_OK) +async def root(request: Request): + logger.debug(f"Request: {request.method} {request.url.path}") + ... +``` +**Why it matters**: +- Helps trace API usage during development. +- Useful for debugging routing problems and unexpected client behavior. + +## 4. Dependencies (requirements.txt) +The project keeps dependencies minimal and focused on what is required to run a FastAPI service in production. +### requirements.txt +```txt +fastapi==0.122.0 +uvicorn[standard]==0.38.0 +``` +**Why it matters**: +- Faster builds & simpler setup: fewer packages mean faster installation and fewer moving parts. +- Lower risk of conflicts: minimal dependencies reduce version incompatibilities and β€œdependency hell”. +- Better security posture: fewer third-party libraries reduce the overall attack surface. +- More predictable deployments: only installing what the service truly needs improves reproducibility across environments (local, CI, Docker, VM). + +## 5. Git Ignore (.gitignore) + +A `.gitignore` file is used to prevent committing temporary, machine-specific, or sensitive files into the repository. + +### Recommended `.gitignore` +```gitignore +# Python +__pycache__/ +*.py[cod] +venv/ +*.log + +# IDE +.vscode/ +.idea/ + +# OS +.DS_Store +``` +**Why it matters**: +- Keeps the repository clean: avoids committing generated files (`__pycache__`, build outputs, logs). +- Improves portability: prevents OS- and IDE-specific files from polluting the project and causing noisy diffs. +- Protects secrets: ensures configuration files like `.env` (which may contain API keys or credentials) are not accidentally pushed. +- Reduces merge conflicts: fewer irrelevant files tracked by Git means fewer conflicts between contributors. + +# 3. API Documentation +The service exposes two endpoints: the main information endpoint and a health check endpoint. +## Request/response examples +### GET `/` β€” Service and System Information +**Description:** +Returns comprehensive metadata about the service, system, runtime, request details, and available endpoints. +**Request example:** +```bash +curl -i http://127.0.0.1:8000/ +``` +**Response example (200 OK):** +```json +{ + "service": { + "name": "devops-info-service", + "version": "1.0.0", + "description": "DevOps course info service", + "framework": "FastAPI" + }, + "system": { + "hostname": "my-laptop", + "platform": "Linux", + "platform_version": "Ubuntu 24.04", + "architecture": "x86_64", + "cpu_count": 8, + "python_version": "3.13.1" + }, + "runtime": { + "uptime_seconds": 3600, + "uptime_human": "1 hour, 0 minutes", + "current_time": "2026-01-07T14:30:00.000Z", + "timezone": "UTC" + }, + "request": { + "client_ip": "127.0.0.1", + "user_agent": "curl/7.81.0", + "method": "GET", + "path": "/" + }, + "endpoints": [ + {"path": "/", "method": "GET", "description": "Service information"}, + {"path": "/health", "method": "GET", "description": "Health check"} + ] +} +``` +### GET /health β€” Health Check +**Description:** +Returns a simple status response to confirm the service is running.**Request example:** +```bash +curl -i http://127.0.0.1:8000/health +``` +**Response example (200 OK):** +```json +{ + "status": "healthy", + "timestamp": "2024-01-15T14:30:00.000Z", + "uptime_seconds": 3600 +} +``` +## Testing commands +### Basic tests +```bash +curl http://127.0.0.1:8000/ +curl http://127.0.0.1:8000/health +``` +### Test 404 handling (unknown endpoint) +```bash +curl -i http://127.0.0.1:8000/does-not-exist +``` +Expected response (404): +```json +{ + "error": "Not Found", + "message": "Endpoint does not exist" +} +``` + +# 4. Testing Evidence +Check screenshots. + +# 5. Challenges & Solutions +I have no problems in this lab. + +# GitHub Community +**Why Stars Matter:** + +**Discovery & Bookmarking:** +- Stars help you bookmark interesting projects for later reference +- Star count indicates project popularity and community trust +- Starred repos appear in your GitHub profile, showing your interests + +**Open Source Signal:** +- Stars encourage maintainers (shows appreciation) +- High star count attracts more contributors +- Helps projects gain visibility in GitHub search and recommendations + +**Professional Context:** +- Shows you follow best practices and quality projects +- Indicates awareness of industry tools and trends + +**Why Following Matters:** + +**Networking:** +- See what other developers are working on +- Discover new projects through their activity +- Build professional connections beyond the classroom + +**Learning:** +- Learn from others' code and commits +- See how experienced developers solve problems +- Get inspiration for your own projects + +**Collaboration:** +- Stay updated on classmates' work +- Easier to find team members for future projects +- Build a supportive learning community + +**Career Growth:** +- Follow thought leaders in your technology stack +- See trending projects in real-time +- Build visibility in the developer community \ No newline at end of file diff --git a/app_python/docs/LAB02.md b/app_python/docs/LAB02.md new file mode 100644 index 0000000000..977e742f10 --- /dev/null +++ b/app_python/docs/LAB02.md @@ -0,0 +1,243 @@ +# Docker Best Practices Applied + +## 1. Non-root user +**What I did:** +Created a dedicated user (`appuser`) and ran the container as that user. + +**Why it matters**: +Running as root in a container increases the impact of a container escape or a vulnerable dependency. A non-root user reduces privileges and limits potential damage. + +**Dockerfile snippet:** +```dockerfile +RUN useradd --create-home --shell /bin/bash appuser +USER appuser +``` + +## 2. Layer caching +**What I did:** +Copied `requirements.txt` first and installed dependencies before copying the rest of the source code. + +**Why it matters**: +Docker caches layers. Dependencies change less frequently than application code, so separating them allows rebuilds to reuse the cached dependency layer and rebuild faster. + +**Dockerfile snippet:** +```dockerfile +COPY requirements.txt . +RUN pip install --no-cache-dir -r requirements.txt +COPY app.py . +``` + +## 3. .dockerignore +**What I did:** +Added .dockerignore to exclude local artifacts such as venvs, caches, and IDE folders. + +**Why it matters**: +Docker sends the build context to the daemon. Excluding unnecessary files makes builds faster, reduces image bloat, and prevents leaking local secrets/files into the build. + +**.dockerignore snippet:** +```.dockerignore +# πŸ™ Version control +.git +.gitignore + +# 🐍 Python +__pycache__ +*.pyc +*.pyo +venv/ +.venv/ + +# πŸ” Secrets (NEVER include!) +.env +*.pem +secrets/ + +# πŸ“ Documentation +*.md +docs/ + +# πŸ§ͺ Tests (if not needed in container) +tests/ +``` + +## 4. Minimal base image +**What I did:** +Used `python:3.12-slim`. + +**Why it matters**: +Smaller images generally mean fewer packages, fewer vulnerabilities, less bandwidth and faster pulls/deployments. + +**dockerfile snippet:** +```dockerfile +FROM python:3.12-slim +``` + +# Image Information & Decisions +## Base image chosen and justification (why this specific version?) +**Base image**: +`python:3.12-slim` + +**Justification**: +- Python 3.12 matches the project runtime requirements. +- slim variant keeps image smaller and reduces OS packages. +- Official Python images are widely used and well maintained. + +## Final image size and my assessment +**Image size**: +`195MB` + +**My assessment**: +Further optimization is possible (multi-stage build, wheels caching, removing build deps), but for this lab the size is acceptable. + +## Layer structure explanation +1. Base image layer (`python:3.12-slim`) +2. User creation layer +3. `WORKDIR` and `requirements.txt` copy layer +4. pip install dependencies layer (largest and most valuable for caching) +5. Application source copy layer +6. EXPOSE, USER, and CMD metadata layers + +## Optimization choices +- Used dependency-first copying to maximize caching. +- Used `--no-cache-dir` with pip to reduce layer size. +- Used slim base image to reduce OS footprint. +- Added `.dockerignore` to reduce build context. + +# Build & Run Process +## Complete terminal output from build process +```bash + docker build -t python_app:1.0 . +[+] Building 5.4s (12/12) FINISHED docker:desktop-linux + => [internal] load build definition from Dockerfile 0.0s + => => transferring dockerfile: 277B 0.0s + => [internal] load metadata for docker.io/library/python:3.12-slim 3.7s + => [auth] library/python:pull token for registry-1.docker.io 0.0s + => [internal] load .dockerignore 0.0s + => => transferring context: 289B 0.0s + => [1/6] FROM docker.io/library/python:3.12-slim@sha256:5e2dbd4bbdd9c0e67412aea9463906f74a22c60f89e 0.0s + => [internal] load build context 0.1s + => => transferring context: 62.28kB 0.1s + => CACHED [2/6] RUN useradd --create-home --shell /bin/bash appuser 0.0s + => CACHED [3/6] WORKDIR /app 0.0s + => CACHED [4/6] COPY requirements.txt . 0.0s + => CACHED [5/6] RUN pip install --no-cache-dir -r requirements.txt 0.0s + => [6/6] COPY app.py . 0.8s + => exporting to image 0.4s + => => exporting layers 0.3s + => => writing image sha256:a3d1dd41a468a1bb53d02edd846964c240eb160f49fd28e9f6ad90fc15677c52 0.0s + => => naming to docker.io/library/python_app:1.0 0.0s + +View build details: docker-desktop://dashboard/build/desktop-linux/desktop-linux/djiu836gkk9g7syuakivz5ynt +``` + +## Terminal output showing container running +```bash + docker run --rm -p 8000:8000 python_app:1.0 +2026-01-31 18:29:56,977 - __main__ - INFO - Application starting... +INFO: Will watch for changes in these directories: ['/app'] +INFO: Uvicorn running on http://0.0.0.0:8000 (Press CTRL+C to quit) +INFO: Started reloader process [1] using WatchFiles +INFO: Started server process [8] +INFO: Waiting for application startup. +INFO: Application startup complete. +``` + +## Terminal output from testing endpoints +```bash +curl http://localhost:8000/ + + StatusCode : 200 +StatusDescription : OK +Content : {"service":{"name":"devops-info-service","version":"1.0.0","description":"DevOps cours + e info service","framework":"FastAPI"},"system":{"hostname":"3a77a23940f7","platform": + "Linux","platform_version":"... +RawContent : HTTP/1.1 200 OK + Content-Length: 755 + Content-Type: application/json + Date: Sat, 31 Jan 2026 18:31:18 GMT + Server: uvicorn + +Forms : {} +Headers : {[Content-Length, 755], [Content-Type, application/json], [Date, Sat, 31 Jan 2026 18:3 + 1:18 GMT], [Server, uvicorn]} Images : {} InputFields : {} Links : {} +ParsedHtml : mshtml.HTMLDocumentClass +RawContentLength : 755 +``` + +```bash +curl http://localhost:8000/health + + +StatusCode : 200 +StatusDescription : OK +Content : {"status":"healthy","timestamp":"2026-01-31T18:31:26.924513+00:00","uptime_seconds":89 + } +RawContent : HTTP/1.1 200 OK + Content-Length: 87 + Content-Type: application/json + Date: Sat, 31 Jan 2026 18:31:26 GMT + Server: uvicorn + + {"status":"healthy","timestamp":"2026-01-31T18:31:26.924513+00:00","uptime_... +Forms : {} +Headers : {[Content-Length, 87], [Content-Type, application/json], [Date, Sat, 31 Jan 2026 18:31 + :26 GMT], [Server, uvicorn]} +Images : {} +InputFields : {} +Links : {} +ParsedHtml : mshtml.HTMLDocumentClass +RawContentLength : 87 +``` + +## Terminal output showing successful push: +```bash +docker push newspec/python_app:1.0 + +The push refers to repository [docker.io/newspec/python_app] +8422fdf98022: Pushed +56a7b3684a2c: Pushed +410b7369101c: Pushed +4e7298e95b69: Pushed +b68196304589: Pushed +343fbb74dfa7: Pushed +cfdc6d123592: Pushed +ff565e4de379: Pushed +e50a58335e13: Pushed +1.0: digest: sha256:9084f1513bc5af085a268ee9e8b165af82f7224e442da0790cf81f07b67ab10e size: 2203 + +``` + +## Docker Hub repository URL +`https://hub.docker.com/repository/docker/newspec/python_app` + +## My tagging strategy +`:1.0`(major/minor) + +# Technical analysis +## Why does your Dockerfile work the way it does? +- The image is based on a minimal Python runtime. +- Dependencies are installed before application code to leverage Docker layer caching. +- The app runs as a non-root user for better security. + +## What would happen if you changed the layer order? +If the Dockerfile copied the entire project (`COPY . .`) before installing requirements, then any code change would invalidate the cache for the dependency install layer. That would force `pip install` to run again on every rebuild, making builds much slower. + +## What security considerations did you implement? +- Non-root execution reduces privilege. +- Smaller base image reduces attack surface. +- `.dockerignore` prevents accidentally shipping local files (including potential secrets) into the image. + +## How does .dockerignore improve your build? +- Reduces build context size (faster build, less IO). +- Avoids copying local venvs and cache files into the image. +- Prevents leaking IDE configs, git history, logs, and other irrelevant files. + +## Challenges & Solutions +# 1. β€œI cannot open my application” after `docker run` +**Fix**: Published the port with `-p 8000:8000` + +# What I Learned +- Containers need explicit port publishing to be accessible from the host. +- Layer ordering dramatically affects build speed due to caching. +- Running as non-root is a simple but important security improvement. +- .dockerignore is crucial to keep images clean and builds fast. \ No newline at end of file diff --git a/app_python/docs/LAB03.md b/app_python/docs/LAB03.md new file mode 100644 index 0000000000..09c3e39c28 --- /dev/null +++ b/app_python/docs/LAB03.md @@ -0,0 +1,170 @@ +# LAB03 β€” Unit Testing + CI/CD + Security + +## 1. Overview + +### Testing framework used and why you chose it + +I chose **pytest** after comparing common Python testing frameworks (`unittest`, `pytest`, etc.). +Pytest requires less boilerplate, has powerful fixtures and parametrization, produces clear failure +output, and scales well with plugins and CI workflows. + +### What endpoints/functionality your tests cover + +All tests are located in `app_python/tests/` and use FastAPI’s `TestClient`. The test suite covers: + +- **GET /** + Verifies status code `200`, JSON structure, required top-level + sections (`service`, `system`, `runtime`, `request`, `endpoints`), and important nested + fields/types. +- **GET /health** + Verifies status code `200`, required fields (`status`, `timestamp`, `uptime_seconds`), and basic + format checks. +- **Error cases** + - **404 Not Found** returns the custom + JSON `{ "error": "Not Found", "message": "Endpoint does not exist" }` + - **Non-404 HTTPException** returns `{ "error": "HTTP Error", "message": "" }` + - **500 Internal Server Error** + returns `{ "error": "Internal Server Error", "message": "An unexpected error occurred" }` + +### CI workflow trigger configuration (when does it run?) + +The GitHub Actions workflow runs on **push** and **pull requests** to `lab03` and `master`, but only +when changes affect: + +- `app_python/**` +- `.github/workflows/python-ci.yml` + +This avoids unnecessary CI runs for unrelated edits. Docker images are built and pushed only on * +*push** events (not on pull requests). + +### Versioning strategy chosen and rationale + +I use **CalVer (Calendar Versioning)** with format `YYYY.MM.DD` because this project is updated +frequently and doesn’t require manual git release tags. Date-based versions are simple, +human-readable, and work well for continuous delivery. + +--- + +## 2. Workflow Evidence + +### βœ… Successful workflow run (GitHub Actions link) + +- https://github.com/newspec/DevOps-Core-Course/actions/runs/21822195126 + +### βœ… Tests passing locally (terminal output) + +```bash +pytest +========================================== test session starts =========================================== +platform win32 -- Python 3.12.4, pytest-8.4.2, pluggy-1.6.0 +rootdir: C:\Users\malov\PycharmProjects\DevOps-Core-Course +plugins: anyio-4.11.0, asyncio-1.3.0, cov-7.0.0 +asyncio: mode=Mode.STRICT, debug=False, asyncio_default_fixture_loop_scope=None, asyncio_default_test_loop_scope=function +collected 7 items + +app_python\tests\test_errors.py ... [ 42%] +app_python\tests\test_health.py .. [ 71%] +app_python\tests\test_root.py .. [100%] + +=========================================== 7 passed in 0.48s ============================================ +``` + +### βœ… Docker image on Docker Hub (link to your image) + +- https://hub.docker.com/repository/docker/newspec/python_app/general + +### βœ… Status badge working in README + +Check the top page of README.md + +## 3. Best Practices Implemented + +- **Fail Fast**: If a step fails, the job stops immediately, saving CI time and making failures + obvious. + +- **Job Dependencies** : Docker push depends on successful `test` and `security` jobs, preventing + publishing broken/insecure builds. + +- **Dependency Caching (pip)**: `setup-python` caches pip downloads so installs are faster on + repeated runs. + +- **Docker Layer Caching**: Buildx + GHA cache reuses Docker layers across runs, reducing Docker + build time significantly. + +- **Secrets Management**: Tokens (Docker Hub + Snyk) are stored in GitHub Secrets and never + committed. + +### Caching: time saved (before vs after) + +Measured by comparing two workflow runs: + +- **Cold run (cache miss)**: 80s total + - tests: 14s + - security: 29s + - docker build: 41s + +- **Warm run (cache hit)**: 64s total + - tests: 11s + - security: 22s + - docker build: 30s + +**Improvement**: + +- Total time saved: **16s** +- Docker build time saved: **11s** +- Percent improvement: **~20%** + +Evidence (screenshots): + +- First run (no cache): ![cache_miss.png](screenshots/cache_miss.png) +- Second run (cache hit): ![cache_hit.png](screenshots/cache_hit.png) + +### Snyk: vulnerabilities found? action taken + +Snyk is executed in a separate `security` job using: + +```bash +snyk test --severity-threshold=high +``` + +- If `high` (or above) vulnerabilities are found, the security job fails. +- Because Docker depends on `security`, the image will **not be pushed** until vulnerabilities are + fixed or the threshold is adjusted. + +**Snyk result**: 0 high/critical vulnerabilities (build passed) + +## Key Decisions + +### Versioning Strategy: SemVer or CalVer? Why did you choose it for your app? + +I chose **CalVer** (`YYYY.MM.DD`) because the application is built frequently and does not follow +formal release cycles. Date-based versioning is easy to automate in CI and provides clear +information about when an image was built. + +### Docker Tags: What tags does your CI create? (e.g., latest, version number, etc.) + +The CI publishes the Docker image with these tags: + +- `YYYY.MM.DD` (CalVer date tag) β€” e.g., `2026.02.09` +- `${{ github.sha }}` (commit SHA tag) β€” uniquely identifies the build source commit +- `latest` β€” points to the most recent image build from the `lab03` or `master` branch + +### Workflow Triggers: Why did you choose those triggers? + +The workflow runs on push and pull request to `lab03` and `master` and only when relevant files +change. This avoids running CI for unrelated edits. Docker images are built and pushed only on push +events (not on pull requests) for `lab03` and `master`. + +### Test Coverage: What's tested vs not tested? + +**Tested**: + +- Successful responses for `GET /` and `GET /health` +- Response JSON structure and required fields +- Custom error handling for 404, non-404 `HTTPException`, and 500 errors + +**Not tested**: + +- Performance/load behavior +- External integrations (none in this app) +- Detailed validation of dynamic fields beyond basic format/type checks (e.g., exact timestamps) \ No newline at end of file diff --git a/app_python/docs/screenshots/01-main-endpoint.png b/app_python/docs/screenshots/01-main-endpoint.png new file mode 100644 index 0000000000..2b5ad4b17f Binary files /dev/null and b/app_python/docs/screenshots/01-main-endpoint.png differ diff --git a/app_python/docs/screenshots/02-health-check.png b/app_python/docs/screenshots/02-health-check.png new file mode 100644 index 0000000000..321da88450 Binary files /dev/null and b/app_python/docs/screenshots/02-health-check.png differ diff --git a/app_python/docs/screenshots/03-formatted-output.png b/app_python/docs/screenshots/03-formatted-output.png new file mode 100644 index 0000000000..bd76d10670 Binary files /dev/null and b/app_python/docs/screenshots/03-formatted-output.png differ diff --git a/app_python/docs/screenshots/cache_hit.png b/app_python/docs/screenshots/cache_hit.png new file mode 100644 index 0000000000..cf23e1e1a9 Binary files /dev/null and b/app_python/docs/screenshots/cache_hit.png differ diff --git a/app_python/docs/screenshots/cache_miss.png b/app_python/docs/screenshots/cache_miss.png new file mode 100644 index 0000000000..a03f43a091 Binary files /dev/null and b/app_python/docs/screenshots/cache_miss.png differ diff --git a/app_python/requirements.txt b/app_python/requirements.txt new file mode 100644 index 0000000000..541fffb380 --- /dev/null +++ b/app_python/requirements.txt @@ -0,0 +1,7 @@ +fastapi==0.122.0 +uvicorn[standard]==0.38.0 +pytest==8.4.2 +flake8==7.3.0 +httpx==0.28.1 +pytest-cov==7.0.0 +python-json-logger==3.2.1 diff --git a/app_python/tests/__init__.py b/app_python/tests/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/app_python/tests/conftest.py b/app_python/tests/conftest.py new file mode 100644 index 0000000000..d1bf72913b --- /dev/null +++ b/app_python/tests/conftest.py @@ -0,0 +1,14 @@ +import pytest +from fastapi.testclient import TestClient + +from app import app + + +@pytest.fixture() +def client(): + return TestClient(app) + + +@pytest.fixture() +def client_no_raise(): + return TestClient(app, raise_server_exceptions=False) diff --git a/app_python/tests/test_errors.py b/app_python/tests/test_errors.py new file mode 100644 index 0000000000..076f70dff6 --- /dev/null +++ b/app_python/tests/test_errors.py @@ -0,0 +1,52 @@ +from fastapi import HTTPException + +from app import app + + +def test_500_returns_custom_json(client_no_raise): + @app.get("/__test_crash__") + async def __test_crash__(): + raise RuntimeError("boom") + + try: + r = client_no_raise.get("/__test_crash__") + assert r.status_code == 500 + assert r.json() == { + "error": "Internal Server Error", + "message": "An unexpected error occurred", + } + finally: + app.router.routes = [ + route for route in app.router.routes + if getattr(route, "path", None) != "/__test_crash__" + ] + + +def test_http_exception_handler_404_custom_json(client): + r = client.get("/this-endpoint-does-not-exist") + + assert r.status_code == 404 + assert r.json() == { + "error": "Not Found", + "message": "Endpoint does not exist", + } + + +def test_http_exception_handler_non_404_returns_http_error_json(client): + @app.get("/__test_http_418__") + async def __test_http_418__(): + raise HTTPException(status_code=418, detail="I'm a teapot") + + try: + r = client.get("/__test_http_418__") + + assert r.status_code == 418 + assert r.json() == { + "error": "HTTP Error", + "message": "I'm a teapot", + } + finally: + app.router.routes = [ + route for route in app.router.routes + if getattr(route, "path", None) != "/__test_http_418__" + ] diff --git a/app_python/tests/test_health.py b/app_python/tests/test_health.py new file mode 100644 index 0000000000..1d2b4f9dcd --- /dev/null +++ b/app_python/tests/test_health.py @@ -0,0 +1,18 @@ +def test_health_status_code(client): + r = client.get("/health") + assert r.status_code == 200 + + +def test_health_response_structure(client): + r = client.get("/health") + data = r.json() + + for key in ["status", "timestamp", "uptime_seconds"]: + assert key in data, f"Missing health field: {key}" + + assert data["status"] == "healthy" + assert isinstance(data["uptime_seconds"], int) + assert data["uptime_seconds"] >= 0 + + assert isinstance(data["timestamp"], str) + assert "T" in data["timestamp"] diff --git a/app_python/tests/test_root.py b/app_python/tests/test_root.py new file mode 100644 index 0000000000..9ca61460fa --- /dev/null +++ b/app_python/tests/test_root.py @@ -0,0 +1,70 @@ +def test_root_status_code(client): + r = client.get("/") + assert r.status_code == 200 + + +def test_root_json_structure_and_required_fields(client): + r = client.get("/") + data = r.json() + + for key in ["service", "system", "runtime", "request", "endpoints"]: + assert key in data, f"Missing top-level field: {key}" + + service = data["service"] + for key in ["name", "version", "description", "framework"]: + assert key in service, f"Missing service field: {key}" + assert service["name"] == "devops-info-service" + assert service["framework"] == "FastAPI" + + system = data["system"] + for key in [ + "hostname", + "platform", + "platform_version", + "architecture", + "cpu_count", + "python_version", + ]: + assert key in system, f"Missing system field: {key}" + + assert isinstance(system["hostname"], str) and system["hostname"] + assert isinstance(system["platform"], str) and system["platform"] + assert isinstance(system["architecture"], str) and system["architecture"] + assert ((system["cpu_count"] is None) or + isinstance(system["cpu_count"], int)) + + runtime = data["runtime"] + for key in ["uptime_seconds", "uptime_human", "current_time", "timezone"]: + assert key in runtime, f"Missing runtime field: {key}" + + assert isinstance(runtime["uptime_seconds"], int) + assert runtime["uptime_seconds"] >= 0 + assert isinstance(runtime["uptime_human"], str) and runtime["uptime_human"] + assert runtime["timezone"] == "UTC" + + assert isinstance(runtime["current_time"], str) + assert "T" in runtime["current_time"] + + req = data["request"] + for key in ["client_ip", "user_agent", "method", "path"]: + assert key in req, f"Missing request field: {key}" + + assert req["method"] == "GET" + assert req["path"] == "/" + assert isinstance(req["client_ip"], str) and req["client_ip"] + assert ("user_agent" in req) + + endpoints = data["endpoints"] + assert isinstance(endpoints, list) + assert len(endpoints) >= 2 + + paths = {(e.get("path"), e.get("method")) for e in endpoints} + assert ("/", "GET") in paths + assert ("/health", "GET") in paths + + for e in endpoints: + for key in ["path", "method", "description"]: + assert key in e + assert isinstance(e["path"], str) and e["path"].startswith("/") + assert e["method"] in {"GET", "POST", "PUT", "DELETE", "PATCH"} + assert isinstance(e["description"], str) and e["description"] diff --git a/docs/LAB04.md b/docs/LAB04.md new file mode 100644 index 0000000000..b17667b00d --- /dev/null +++ b/docs/LAB04.md @@ -0,0 +1,1192 @@ +# Lab 04 - Infrastructure as Code (Terraform & Pulumi) + +## 1. Cloud Provider & Infrastructure + +### Cloud Provider Choice: Yandex Cloud + +**Rationale:** +- **Accessibility in Russia**: No restrictions or sanctions affecting access +- **Free Tier**: 1 VM with 20% vCPU, 1 GB RAM, 10 GB storage +- **No Credit Card Required**: Can start without payment method +- **Good Documentation**: Available in Russian and English +- **Terraform & Pulumi Support**: Official providers available +- **Local Data Centers**: Lower latency for Russian users + +**Alternative Considered:** +- AWS: More popular globally but requires credit card and may have access issues +- GCP: Good free tier but complex setup +- VK Cloud: Russian alternative but less mature tooling support + +### Instance Configuration + +**VM Specifications:** +- **Platform**: standard-v2 +- **CPU**: 2 cores @ 20% (free tier) +- **Memory**: 1 GB RAM +- **Disk**: 10 GB HDD (network-hdd) +- **OS**: Ubuntu 24.04 LTS +- **Region/Zone**: ru-central1-a + +**Network Configuration:** +- **VPC Network**: Custom network (10.128.0.0/24) +- **Public IP**: Yes (NAT enabled) +- **Security Group Rules**: + - SSH (port 22): Allow from anywhere (0.0.0.0/0) + - HTTP (port 80): Allow from anywhere + - Custom (port 5000): Allow from anywhere (for app deployment) + - Egress: Allow all outbound traffic + +### Cost Analysis + +**Total Cost: $0.00/month** + +Using Yandex Cloud free tier: +- VM: Free (20% vCPU, 1 GB RAM within limits) +- Storage: Free (10 GB HDD within limits) +- Network: Free (within egress limits) +- Public IP: Free (1 static IP included) + +### Resources Created + +**Terraform Resources:** +1. `yandex_vpc_network.lab04_network` - VPC network +2. `yandex_vpc_subnet.lab04_subnet` - Subnet (10.128.0.0/24) +3. `yandex_vpc_security_group.lab04_sg` - Security group with firewall rules +4. `yandex_compute_instance.lab04_vm` - VM instance + +**Pulumi Resources:** +1. `lab04-network` - VPC network +2. `lab04-subnet` - Subnet (10.128.0.0/24) +3. `lab04-sg` - Security group with firewall rules +4. `lab04-vm` - VM instance + +--- + +## 2. Terraform Implementation + +### Terraform Version + +```bash +Terraform v1.9.0 +on darwin_arm64 ++ provider registry.terraform.io/integrations/github v5.45.0 ++ provider registry.terraform.io/yandex-cloud/yandex v0.187.0 +``` + +### Project Structure + +``` +terraform/ +β”œβ”€β”€ .terraform.lock.hcl # Provider version lock file +β”œβ”€β”€ .terraformrc # Terraform CLI configuration (Yandex mirror) +β”œβ”€β”€ .tflint.hcl # TFLint configuration +β”œβ”€β”€ main.tf # Main resources (VM, network, security) +β”œβ”€β”€ variables.tf # Input variable declarations +β”œβ”€β”€ outputs.tf # Output value definitions +β”œβ”€β”€ github.tf # GitHub provider (bonus task) +β”œβ”€β”€ terraform.tfvars # Actual configuration (gitignored) +β”œβ”€β”€ terraform.tfstate # State file (gitignored) +└── terraform.tfstate.backup # State backup (gitignored) +``` + +**Note:** `terraform.tfstate` files are present locally but excluded from Git via `.gitignore`. + +### Key Configuration Decisions + +**1. Provider Configuration** +- Used Yandex Cloud provider version ~> 0.187 +- Authentication via Service Account key (authorized key JSON file) +- Configured default zone (ru-central1-a) and folder_id +- GitHub provider version ~> 5.45 for repository management + +**2. Resource Organization** +- Separated resources logically in main.tf +- Used data source for Ubuntu image (latest 24.04 LTS) +- Created dedicated VPC network instead of using default + +**3. Security Approach** +- Security group with explicit ingress/egress rules +- SSH key injection via metadata +- All sensitive values in gitignored terraform.tfvars +- Used variables for all configurable parameters + +**4. Free Tier Optimization** +- Set `core_fraction = 20` for free tier CPU +- Used `network-hdd` disk type (cheaper than SSD) +- Minimal 10 GB disk size +- Single VM instance + +### Challenges Encountered + +**1. Authentication Setup** +- **Issue**: Initial confusion about OAuth token vs service account +- **Solution**: Created Service Account with appropriate roles, generated authorized key (JSON) +- **Learning**: Service accounts provide better security and are recommended for automation + +**3. Image Selection** +- **Issue**: Needed to find correct Ubuntu 24.04 image family name +- **Solution**: Used data source with `family = "ubuntu-2404-lts"` +- **Learning**: Data sources are powerful for dynamic resource lookup + +**4. Free Tier Configuration** +- **Issue**: Ensuring configuration stays within free tier limits +- **Solution**: Set `core_fraction = 20`, used network-hdd, 10 GB disk +- **Learning**: Important to understand cloud provider pricing models + +### Terraform Commands Output + +#### terraform init + +```bash +$ cd terraform/ +terraform init + +Initializing the backend... + +Initializing provider plugins... +- Finding yandex-cloud/yandex versions matching "~> 0.100"... +- Finding integrations/github versions matching "~> 5.0"... +- Installing yandex-cloud/yandex v0.187.0... +- Installed yandex-cloud/yandex v0.187.0 (unauthenticated) +- Installing integrations/github v5.45.0... +- Installed integrations/github v5.45.0 (unauthenticated) + +Terraform has created a lock file .terraform.lock.hcl to record the provider +selections it made above. Include this file in your version control repository +so that Terraform can guarantee to make the same selections by default when +you run "terraform init" in the future. + +β•· +β”‚ Warning: Incomplete lock file information for providers +β”‚ +β”‚ Due to your customized provider installation methods, Terraform was forced to calculate lock file checksums +β”‚ locally for the following providers: +β”‚ - integrations/github +β”‚ - yandex-cloud/yandex +β”‚ +β”‚ The current .terraform.lock.hcl file only includes checksums for darwin_arm64, so Terraform running on another +β”‚ platform will fail to install these providers. +β”‚ +β”‚ To calculate additional checksums for another platform, run: +β”‚ terraform providers lock -platform=linux_amd64 +β”‚ (where linux_amd64 is the platform to generate) +β•΅ + +Terraform has been successfully initialized! + +You may now begin working with Terraform. Try running "terraform plan" to see +any changes that are required for your infrastructure. All Terraform commands +should now work. + +If you ever set or change modules or backend configuration for Terraform, +rerun this command to reinitialize your working directory. If you forget, other +commands will detect it and remind you to do so if necessary. +``` + +#### terraform plan + +```bash +terraform plan +var.cloud_id + Yandex Cloud ID + + Enter a value: ******** + +data.yandex_compute_image.ubuntu: Reading... +data.yandex_compute_image.ubuntu: Read complete after 0s [id=********] + +Terraform used the selected providers to generate the following execution plan. Resource actions are indicated with the following symbols: + + create + +Terraform will perform the following actions: + + # github_branch_protection.master_protection will be created + + resource "github_branch_protection" "master_protection" { + + allows_deletions = false + + allows_force_pushes = false + + blocks_creations = false + + enforce_admins = false + + id = (known after apply) + + lock_branch = false + + pattern = "master" + + repository_id = (known after apply) + + require_conversation_resolution = false + + require_signed_commits = false + + required_linear_history = false + + + required_pull_request_reviews { + + dismiss_stale_reviews = true + + require_code_owner_reviews = false + + require_last_push_approval = false + + required_approving_review_count = 0 + } + } + + # github_repository.devops_course will be created + + resource "github_repository" "devops_course" { + + allow_auto_merge = false + + allow_merge_commit = true + + allow_rebase_merge = true + + allow_squash_merge = true + + archived = false + + default_branch = (known after apply) + + delete_branch_on_merge = true + + description = "DevOps Engineering: Core Practices - Lab assignments and projects" + + etag = (known after apply) + + full_name = (known after apply) + + git_clone_url = (known after apply) + + has_downloads = true + + has_issues = true + + has_projects = false + + has_wiki = false + + html_url = (known after apply) + + http_clone_url = (known after apply) + + id = (known after apply) + + merge_commit_message = "PR_TITLE" + + merge_commit_title = "MERGE_MESSAGE" + + name = "DevOps-Core-Course" + + node_id = (known after apply) + + primary_language = (known after apply) + + private = (known after apply) + + repo_id = (known after apply) + + squash_merge_commit_message = "COMMIT_MESSAGES" + + squash_merge_commit_title = "COMMIT_OR_PR_TITLE" + + ssh_clone_url = (known after apply) + + svn_url = (known after apply) + + topics = [ + + "ansible", + + "ci-cd", + + "devops", + + "docker", + + "infrastructure-as-code", + + "kubernetes", + + "pulumi", + + "terraform", + ] + + visibility = "public" + + web_commit_signoff_required = false + } + + # yandex_compute_instance.lab04_vm will be created + + resource "yandex_compute_instance" "lab04_vm" { + + created_at = (known after apply) + + folder_id = (known after apply) + + fqdn = (known after apply) + + gpu_cluster_id = (known after apply) + + hardware_generation = (known after apply) + + hostname = "lab04-vm" + + id = (known after apply) + + labels = { + + "environment" = "lab04" + + "managed_by" = "terraform" + + "purpose" = "devops-course" + } + + maintenance_grace_period = (known after apply) + + maintenance_policy = (known after apply) + + metadata = { + + "ssh-keys" = <<-EOT + ************************ + EOT + } + + name = "lab04-vm" + + network_acceleration_type = "standard" + + platform_id = "standard-v2" + + status = (known after apply) + + zone = "ru-central1-a" + + + boot_disk { + + auto_delete = true + + device_name = (known after apply) + + disk_id = (known after apply) + + mode = (known after apply) + + + initialize_params { + + block_size = (known after apply) + + description = (known after apply) + + image_id = "fd8lt661chfo5i13a40d" + + name = (known after apply) + + size = 10 + + snapshot_id = (known after apply) + + type = "network-hdd" + } + } + + + network_interface { + + index = (known after apply) + + ip_address = (known after apply) + + ipv4 = true + + ipv6 = (known after apply) + + ipv6_address = (known after apply) + + mac_address = (known after apply) + + nat = true + + nat_ip_address = (known after apply) + + nat_ip_version = (known after apply) + + subnet_id = (known after apply) + } + + + resources { + + core_fraction = 20 + + cores = 2 + + memory = 1 + } + + + scheduling_policy { + + preemptible = false + } + } + + # yandex_vpc_network.lab04_network will be created + + resource "yandex_vpc_network" "lab04_network" { + + created_at = (known after apply) + + default_security_group_id = (known after apply) + + description = "Network for Lab 04 VM" + + folder_id = (known after apply) + + id = (known after apply) + + labels = (known after apply) + + name = "lab04-network" + + subnet_ids = (known after apply) + } + + # yandex_vpc_subnet.lab04_subnet will be created + + resource "yandex_vpc_subnet" "lab04_subnet" { + + created_at = (known after apply) + + description = "Subnet for Lab 04 VM" + + folder_id = (known after apply) + + id = (known after apply) + + labels = (known after apply) + + name = "lab04-subnet" + + network_id = (known after apply) + + v4_cidr_blocks = [ + + "10.128.0.0/24", + ] + + v6_cidr_blocks = (known after apply) + + zone = "ru-central1-a" + } + +Plan: 5 to add, 0 to change, 0 to destroy. + +Changes to Outputs: + + connection_info = { + + private_ip = (known after apply) + + public_ip = (known after apply) + + ssh_command = (known after apply) + + ssh_user = "ubuntu" + } + + network_id = (known after apply) + + ssh_command = (known after apply) + + subnet_id = (known after apply) + + vm_id = (known after apply) + + vm_name = "lab04-vm" + + vm_private_ip = (known after apply) + + vm_public_ip = (known after apply) + +────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────── + +Note: You didn't use the -out option to save this plan, so Terraform can't guarantee to take exactly these actions if you run "terraform apply" now. +``` + +#### terraform apply + +```bash +terraform apply +var.cloud_id + Yandex Cloud ID + + Enter a value: ****** + +github_repository.devops_course: Refreshing state... [id=DevOps-Core-Course] +data.yandex_compute_image.ubuntu: Reading... +yandex_vpc_network.lab04_network: Refreshing state... [id=******] +data.yandex_compute_image.ubuntu: Read complete after 0s [id=******] +yandex_vpc_subnet.lab04_subnet: Refreshing state... [id=******] +yandex_compute_instance.lab04_vm: Refreshing state... [id=******] +github_branch_protection.master_protection: Refreshing state... [id=B******] + +Terraform used the selected providers to generate the following execution plan. Resource actions are indicated with +the following symbols: +-/+ destroy and then create replacement + +Terraform will perform the following actions: + + # github_branch_protection.master_protection is tainted, so must be replaced +-/+ resource "github_branch_protection" "master_protection" { + - force_push_bypassers = [] -> null + ~ id = "******" -> (known after apply) + - push_restrictions = [] -> null + # (10 unchanged attributes hidden) + + ~ required_pull_request_reviews { + - dismissal_restrictions = [] -> null + - pull_request_bypassers = [] -> null + - restrict_dismissals = false -> null + # (4 unchanged attributes hidden) + } + } + +Plan: 1 to add, 0 to change, 1 to destroy. + +Do you want to perform these actions? + Terraform will perform the actions described above. + Only 'yes' will be accepted to approve. + + Enter a value: yes + +github_branch_protection.master_protection: Destroying... [id=******] +github_branch_protection.master_protection: Destruction complete after 0s +github_branch_protection.master_protection: Creating... +github_branch_protection.master_protection: Creation complete after 4s [id=******] + +Apply complete! Resources: 1 added, 0 changed, 1 destroyed. + +Outputs: + +connection_info = { + "private_ip" = "10.128.0.11" + "public_ip" = "84.201.128.171" + "ssh_command" = "ssh ubuntu@84.201.128.171" + "ssh_user" = "ubuntu" +} +network_id = "enp5kqg9rma6c31bjsen" +ssh_command = "ssh ubuntu@84.201.128.171" +subnet_id = "e9bl6fnifjfbe7ufp7tl" +vm_id = "fhmbajpub1spksjhkvct" +vm_name = "lab04-vm" +vm_private_ip = "10.128.0.11" +vm_public_ip = "84.201.128.171" +``` + +### SSH Connection Verification + +```bash +ssh -i ~/.ssh/yandex_cloud_key ubuntu@84.201.128.171 +Welcome to Ubuntu 24.04.4 LTS (GNU/Linux 6.8.0-100-generic x86_64) + + * Documentation: https://help.ubuntu.com + * Management: https://landscape.canonical.com + * Support: https://ubuntu.com/pro + + System information as of Mon Feb 16 23:12:16 UTC 2026 + + System load: 0.0 Processes: 96 + Usage of /: 23.1% of 9.04GB Users logged in: 0 + Memory usage: 17% IPv4 address for eth0: 10.128.0.11 + Swap usage: 0% + + +Expanded Security Maintenance for Applications is not enabled. + +0 updates can be applied immediately. + +Enable ESM Apps to receive additional future security updates. +See https://ubuntu.com/esm or run: sudo pro status + + + +The programs included with the Ubuntu system are free software; +the exact distribution terms for each program are described in the +individual files in /usr/share/doc/*/copyright. + +Ubuntu comes with ABSOLUTELY NO WARRANTY, to the extent permitted by +applicable law. + +To run a command as administrator (user "root"), use "sudo ". +See "man sudo_root" for details. + +ubuntu@lab04-vm:~$ +``` + +--- + +## 3. Pulumi Implementation + +### Pulumi Version and Language + +```bash +Pulumi v3.220.0 +Python 3.9.6 +pulumi-yandex v0.13.0 +``` + +**Language Choice: Python** + +### Code Differences from Terraform + +**1. Language Paradigm** + +**Terraform (Declarative HCL):** +```hcl +resource "yandex_vpc_network" "lab04_network" { + name = "lab04-network" + description = "Network for Lab 04 VM" +} +``` + +**Pulumi (Imperative Python):** +```python +network = yandex.VpcNetwork( + "lab04-network", + name="lab04-pulumi-network", + description="Network for Lab 04 Pulumi VM", + folder_id=folder_id +) +``` + +**Key Differences:** +- Terraform: Resource blocks with attributes +- Pulumi: Object instantiation with constructor arguments +- Terraform: Static configuration +- Pulumi: Can use variables, loops, functions naturally + +**2. Configuration Management** + +**Terraform:** +```hcl +# variables.tf +variable "folder_id" { + description = "Yandex Cloud folder ID" + type = string +} + +# terraform.tfvars +folder_id = "b1g..." +``` + +**Pulumi:** +```python +# __main__.py +config = pulumi.Config() +folder_id = config.require("folder_id") + +# Command line +pulumi config set lab04-pulumi:folder_id b1g... +``` + +**Key Differences:** +- Terraform: Separate variable files +- Pulumi: Config object in code +- Terraform: tfvars files +- Pulumi: Stack-specific YAML files or CLI commands + +**3. Outputs** + +**Terraform:** +```hcl +output "vm_public_ip" { + description = "Public IP address of the VM" + value = yandex_compute_instance.lab04_vm.network_interface[0].nat_ip_address +} +``` + +**Pulumi:** +```python +pulumi.export("vm_public_ip", vm.network_interfaces[0].nat_ip_address) + +# For computed values +pulumi.export("ssh_command", vm.network_interfaces[0].nat_ip_address.apply( + lambda ip: f"ssh {ssh_user}@{ip}" +)) +``` + +**Key Differences:** +- Terraform: Output blocks +- Pulumi: Export function calls +- Pulumi: `.apply()` for working with computed values (Promises) + +**4. Resource Dependencies** + +**Terraform:** +```hcl +# Implicit dependencies through references +resource "yandex_vpc_subnet" "lab04_subnet" { + network_id = yandex_vpc_network.lab04_network.id # Implicit dependency +} +``` + +**Pulumi:** +```python +# Same implicit dependencies through references +subnet = yandex.VpcSubnet( + "lab04-subnet", + network_id=network.id # Implicit dependency +) +``` + +**Key Differences:** +- Both handle dependencies automatically +- Pulumi can use explicit `depends_on` if needed +- Pulumi's type system helps catch errors earlier + +### Advantages Discovered + +**1. Programming Language Features** + + **Loops and Conditionals:** +```python +# Easy to create multiple similar resources +for i in range(3): + subnet = yandex.VpcSubnet(f"subnet-{i}", ...) + +# Conditional resource creation +if config.get_bool("enable_monitoring"): + monitoring = yandex.MonitoringDashboard(...) +``` + + **Functions and Reusability:** +```python +def create_security_rule(port, description): + return yandex.VpcSecurityGroupIngressArgs( + protocol="TCP", + description=description, + v4_cidr_blocks=["0.0.0.0/0"], + port=port + ) + +# Use function to create rules +ingress=[ + create_security_rule(22, "Allow SSH"), + create_security_rule(80, "Allow HTTP"), + create_security_rule(5000, "Allow app port 5000"), +] +``` + + **Error Handling:** +```python +try: + with open(ssh_public_key_path, "r") as f: + ssh_public_key = f.read().strip() +except FileNotFoundError: + raise Exception(f"SSH public key not found at {ssh_public_key_path}") +``` + +**2. IDE Support** + + **Autocomplete:** +- IDE suggests available properties +- Type hints show expected types +- Inline documentation + + **Type Checking:** +- Catch errors before deployment +- Better refactoring support +- Clear error messages + +**3. Testing Capabilities** + + **Unit Tests:** +```python +# Can write unit tests for infrastructure +import unittest +from pulumi import runtime + +class TestInfrastructure(unittest.TestCase): + @pulumi.runtime.test + def test_vm_has_correct_size(self): + # Test infrastructure code + pass +``` + +**4. Secrets Management** + + **Encrypted by Default:** +```bash +pulumi config set --secret github_token ghp_... +# Automatically encrypted in Pulumi.*.yaml +``` + + **No Plain Text in State:** +- Secrets encrypted in state file +- Safer than Terraform's plain text state + +### Challenges Encountered + +**1. Learning Curve** +- **Issue**: Understanding Pulumi's async/promise model (`.apply()`) +- **Solution**: Read documentation on Output types and computed values +- **Learning**: Pulumi's Output type handles async resource creation + +**2. Provider Documentation** +- **Issue**: Yandex Cloud Pulumi provider has less documentation than Terraform +- **Solution**: Referred to Terraform docs and translated to Pulumi syntax +- **Learning**: Terraform has larger community and more examples + +**4. Python Path Issues** +- **Issue**: SSH key path with `~` not expanding correctly +- **Solution**: Added manual path expansion in code +- **Learning**: Need to handle OS-specific path issues in code + +### Pulumi Commands Output + +#### pulumi preview + +```bash +pulumi preview +Enter your passphrase to unlock config/secrets + (set PULUMI_CONFIG_PASSPHRASE or PULUMI_CONFIG_PASSPHRASE_FILE to remember): +Enter your passphrase to unlock config/secrets +Previewing update (dev): + Type Name Plan + + pulumi:pulumi:Stack lab04-pulumi-dev create + + β”œβ”€ yandex:index:VpcNetwork lab04-network create + + β”œβ”€ yandex:index:VpcSubnet lab04-subnet create + + β”œβ”€ yandex:index:VpcSecurityGroup lab04-sg create + + └─ yandex:index:ComputeInstance lab04-vm create + +Outputs: + connection_info: { + private_ip : [unknown] + public_ip : [unknown] + ssh_command: [unknown] + ssh_user : "ubuntu" + } + network_id : [unknown] + ssh_command : [unknown] + subnet_id : [unknown] + vm_id : [unknown] + vm_name : "lab04-pulumi-vm" + vm_private_ip : [unknown] + vm_public_ip : [unknown] + +Resources: + + 5 to create + +(venv) newspec@10 pulumi % +``` + +#### pulumi up + +```bash +pulumi up +Enter your passphrase to unlock config/secrets + (set PULUMI_CONFIG_PASSPHRASE or PULUMI_CONFIG_PASSPHRASE_FILE to remember): +Enter your passphrase to unlock config/secrets +Previewing update (dev): + Type Name Plan + + pulumi:pulumi:Stack lab04-pulumi-dev create + + β”œβ”€ yandex:index:VpcNetwork lab04-network create + + β”œβ”€ yandex:index:VpcSubnet lab04-subnet create + + β”œβ”€ yandex:index:VpcSecurityGroup lab04-sg create + + └─ yandex:index:ComputeInstance lab04-vm create + +Outputs: + connection_info: { + private_ip : [unknown] + public_ip : [unknown] + ssh_command: [unknown] + ssh_user : "ubuntu" + } + network_id : [unknown] + ssh_command : [unknown] + subnet_id : [unknown] + vm_id : [unknown] + vm_name : "lab04-pulumi-vm" + vm_private_ip : [unknown] + vm_public_ip : [unknown] + +Resources: + + 5 to create + +Do you want to perform this update? yes +Updating (dev): + Type Name Status + + pulumi:pulumi:Stack lab04-pulumi-dev created (41s) + + β”œβ”€ yandex:index:VpcNetwork lab04-network created (1s) + + β”œβ”€ yandex:index:VpcSecurityGroup lab04-sg created (1s) + + β”œβ”€ yandex:index:VpcSubnet lab04-subnet created (0.43s) + + └─ yandex:index:ComputeInstance lab04-vm created (38s) + +Outputs: + connection_info: { + private_ip : "10.128.0.13" + public_ip : "84.201.128.246" + ssh_command: "ssh ubuntu@84.201.128.246" + ssh_user : "ubuntu" + } + network_id : "enpej60jp6arufbqcu7g" + ssh_command : "ssh ubuntu@84.201.128.246" + subnet_id : "e9bdpptsdf2nafbj1s10" + vm_id : "fhmvjrq2012fqg0mloc8" + vm_name : "lab04-pulumi-vm" + vm_private_ip : "10.128.0.13" + vm_public_ip : "84.201.128.246" + +Resources: + + 5 created + +Duration: 42s +``` + +### SSH Connection Verification + +```bash +ssh -i ~/.ssh/yandex_cloud_key ubuntu@84.201.128.246 +Welcome to Ubuntu 24.04.4 LTS (GNU/Linux 6.8.0-100-generic x86_64) + + * Documentation: https://help.ubuntu.com + * Management: https://landscape.canonical.com + * Support: https://ubuntu.com/pro + + System information as of Mon Feb 16 23:50:44 UTC 2026 + + System load: 0.01 Processes: 99 + Usage of /: 23.1% of 9.04GB Users logged in: 0 + Memory usage: 17% IPv4 address for eth0: 10.128.0.13 + Swap usage: 0% + + +Expanded Security Maintenance for Applications is not enabled. + +0 updates can be applied immediately. + +Enable ESM Apps to receive additional future security updates. +See https://ubuntu.com/esm or run: sudo pro status + + + +The programs included with the Ubuntu system are free software; +the exact distribution terms for each program are described in the +individual files in /usr/share/doc/*/copyright. + +Ubuntu comes with ABSOLUTELY NO WARRANTY, to the extent permitted by +applicable law. + + +To run a command as administrator (user "root"), use "sudo ". +See "man sudo_root" for details. + +ubuntu@lab04-pulumi-vm:~$ +``` +--- + +## 4. Terraform vs Pulumi Comparison + +### Ease of Learning + +**Terraform: (4/5)** + +**Pros:** +- Simple, declarative syntax +- Easy to understand resource blocks +- Extensive documentation and examples +- Large community with many tutorials +- Consistent patterns across providers + +**Cons:** +- Need to learn HCL syntax +- Limited logic capabilities +- Some concepts (count, for_each) can be confusing + +**Pulumi: (3/5)** + +**Pros:** +- Use familiar programming language +- No new syntax to learn (if you know Python) +- Natural use of variables and functions + +**Cons:** +- Need to understand Output/Promise model +- Async concepts can be confusing +- Less community content and examples +- Requires programming knowledge + +**Winner: Terraform** - Lower barrier to entry, especially for those without programming background. + +### Code Readability + +**Terraform: (5/5)** + +**Pros:** +- Very clear and declarative +- Easy to see what infrastructure will be created +- Consistent structure across all resources +- Self-documenting with descriptions + +**Pulumi: (4/5)** + +**Pros:** +- Familiar Python syntax +- Can add comments and documentation strings +- Type hints improve clarity +- IDE shows inline documentation + +**Cons:** +- More verbose than HCL +- Mixing infrastructure and logic can reduce clarity +- Need to understand Python conventions + +**Winner: Terraform** - More concise and purpose-built for infrastructure. + +### Debugging + +**Terraform: (3/5)** + +**Pros:** +- Clear error messages +- `terraform plan` shows what will change +- Can use `terraform console` for testing expressions +- State file helps understand current state + +**Cons:** +- Limited debugging tools +- Hard to debug complex expressions +- No step-through debugging +- Error messages can be cryptic for complex scenarios + +**Pulumi: (4/5)** + +**Pros:** +- Can use Python debugger (pdb) +- IDE debugging support +- Better error messages with stack traces +- Can add print statements for debugging +- Unit testing capabilities + +**Cons:** +- Async nature can complicate debugging +- Output types require `.apply()` understanding + +**Winner: Pulumi** - Full programming language debugging capabilities. + +### Documentation + +**Terraform: (5/5)** + +**Pros:** +- Extensive official documentation +- Large community with many examples +- Provider documentation in Terraform Registry +- Many tutorials and courses +- Stack Overflow has many answers + +**Cons:** +- Documentation can be overwhelming +- Some providers have better docs than others + +**Pulumi: (3/5)** + +**Pros:** +- Good official documentation +- API reference auto-generated +- Examples in multiple languages +- Good getting started guides + +**Cons:** +- Smaller community +- Fewer third-party tutorials +- Less Stack Overflow content +- Provider docs sometimes less detailed + +**Winner: Terraform** - Much larger ecosystem and community. + +### Use Cases + +**When to Use Terraform:** + + **Simple to Medium Infrastructure** +- Straightforward resource provisioning +- Standard cloud patterns +- Team prefers declarative approach + + **Multi-Cloud Deployments** +- Largest provider ecosystem +- Consistent syntax across clouds +- Mature and stable + + **Compliance and Governance** +- Clear audit trail +- Policy as code (Sentinel) +- Established best practices + + **Team Without Programming Background** +- DevOps/Ops teams +- Infrastructure-focused roles +- Lower learning curve + +**When to Use Pulumi:** + + **Complex Infrastructure Logic** +- Dynamic resource creation +- Complex conditionals +- Advanced transformations + + **Developer-Centric Teams** +- Software engineers managing infrastructure +- Want to use familiar languages +- Need testing capabilities + + **Reusable Components** +- Building infrastructure libraries +- Sharing code via packages +- Higher-level abstractions + + **Better Secrets Management** +- Need encrypted secrets +- Compliance requirements +- Sensitive data handling + +## 5. Lab 5 Preparation & Cleanup + +### VM for Lab 5 + +**Are you keeping your VM for Lab 5?** No + +**What will you use for Lab 5?** Will recreate cloud VM + +**Terrafrom destroy**: +```bash +terraform destroy +var.cloud_id + Yandex Cloud ID + + Enter a value: ******** + +github_repository.devops_course: Refreshing state... [id=DevOps-Core-Course] +data.yandex_compute_image.ubuntu: Reading... +data.yandex_compute_image.ubuntu: Read complete after 0s [id=*******] + +Terraform used the selected providers to generate the following execution plan. Resource actions are indicated with the following symbols: + - destroy + +Terraform will perform the following actions: + + # github_repository.devops_course will be destroyed + - resource "github_repository" "devops_course" { + - allow_auto_merge = false -> null + - allow_merge_commit = true -> null + - allow_rebase_merge = true -> null + - allow_squash_merge = true -> null + - allow_update_branch = false -> null + - archived = false -> null + - auto_init = false -> null + - default_branch = "master" -> null + - delete_branch_on_merge = true -> null + - description = "DevOps Engineering: Core Practices - Lab assignments and projects" -> null + - etag = "W/\"8f88878a50eedec268e373e039998430cbf194a2a9e0c3ff93a27116412b1b69\"" -> null + - full_name = "newspec/DevOps-Core-Course" -> null + - git_clone_url = "git://github.com/newspec/DevOps-Core-Course.git" -> null + - has_discussions = false -> null + - has_downloads = true -> null + - has_issues = true -> null + - has_projects = false -> null + - has_wiki = false -> null + - html_url = "https://github.com/newspec/DevOps-Core-Course" -> null + - http_clone_url = "https://github.com/newspec/DevOps-Core-Course.git" -> null + - id = "DevOps-Core-Course" -> null + - is_template = false -> null + - merge_commit_message = "PR_TITLE" -> null + - merge_commit_title = "MERGE_MESSAGE" -> null + - name = "DevOps-Core-Course" -> null + - node_id = "R_kgDORA7Qvw" -> null + - private = false -> null + - repo_id = 1141821631 -> null + - squash_merge_commit_message = "COMMIT_MESSAGES" -> null + - squash_merge_commit_title = "COMMIT_OR_PR_TITLE" -> null + - ssh_clone_url = "git@github.com:newspec/DevOps-Core-Course.git" -> null + - svn_url = "https://github.com/newspec/DevOps-Core-Course" -> null + - topics = [ + - "ansible", + - "ci-cd", + - "devops", + - "docker", + - "infrastructure-as-code", + - "kubernetes", + - "pulumi", + - "terraform", + ] -> null + - visibility = "public" -> null + - vulnerability_alerts = false -> null + - web_commit_signoff_required = false -> null + + - security_and_analysis { + - secret_scanning { + - status = "enabled" -> null + } + - secret_scanning_push_protection { + - status = "enabled" -> null + } + } + } + +Plan: 0 to add, 0 to change, 1 to destroy. + +Changes to Outputs: + - vm_name = "lab04-vm" -> null + +Do you really want to destroy all resources? + Terraform will destroy all your managed infrastructure, as shown above. + There is no undo. Only 'yes' will be accepted to confirm. + + Enter a value: yes + +github_repository.devops_course: Destroying... [id=DevOps-Core-Course] +β•· +β”‚ Error: DELETE https://api.github.com/repos/newspec/DevOps-Core-Course: 403 Must have admin rights to Repository. [] +β”‚ +β”‚ +β•΅ +``` +**pulumi destroy**: +```bash +pulumi destroy +Enter your passphrase to unlock config/secrets + (set PULUMI_CONFIG_PASSPHRASE or PULUMI_CONFIG_PASSPHRASE_FILE to remember): +Enter your passphrase to unlock config/secrets +Previewing destroy (dev): + Type Name Plan + - pulumi:pulumi:Stack lab04-pulumi-dev delete + - β”œβ”€ yandex:index:VpcNetwork lab04-network delete + - β”œβ”€ yandex:index:ComputeInstance lab04-vm delete + - β”œβ”€ yandex:index:VpcSubnet lab04-subnet delete + - └─ yandex:index:VpcSecurityGroup lab04-sg delete + +Outputs: + - connection_info: { + - private_ip : "10.128.0.13" + - public_ip : "84.201.128.246" + - ssh_command: "ssh ubuntu@84.201.128.246" + - ssh_user : "ubuntu" + } + - network_id : "enpej60jp6arufbqcu7g" + - ssh_command : "ssh ubuntu@84.201.128.246" + - subnet_id : "e9bdpptsdf2nafbj1s10" + - vm_id : "fhmvjrq2012fqg0mloc8" + - vm_name : "lab04-pulumi-vm" + - vm_private_ip : "10.128.0.13" + - vm_public_ip : "84.201.128.246" + +Resources: + - 5 to delete + +Do you want to perform this destroy? yes +Destroying (dev): + Type Name Status + - pulumi:pulumi:Stack lab04-pulumi-dev deleted (0.01s) + - β”œβ”€ yandex:index:ComputeInstance lab04-vm deleted (33s) + - β”œβ”€ yandex:index:VpcSubnet lab04-subnet deleted (4s) + - β”œβ”€ yandex:index:VpcSecurityGroup lab04-sg deleted (0.43s) + - └─ yandex:index:VpcNetwork lab04-network deleted (1s) + +Outputs: + - connection_info: { + - private_ip : "10.128.0.13" + - public_ip : "84.201.128.246" + - ssh_command: "ssh ubuntu@84.201.128.246" + - ssh_user : "ubuntu" + } + - network_id : "enpej60jp6arufbqcu7g" + - ssh_command : "ssh ubuntu@84.201.128.246" + - subnet_id : "e9bdpptsdf2nafbj1s10" + - vm_id : "fhmvjrq2012fqg0mloc8" + - vm_name : "lab04-pulumi-vm" + - vm_private_ip : "10.128.0.13" + - vm_public_ip : "84.201.128.246" + +Resources: + - 5 deleted + +Duration: 40s + +The resources in the stack have been deleted, but the history and configuration associated with the stack are still maintained. +If you want to remove the stack completely, run `pulumi stack rm dev`. +``` + +**screenshot showing resource status:** ![alt text](image.png) diff --git a/docs/image.png b/docs/image.png new file mode 100644 index 0000000000..83ce4aa12f Binary files /dev/null and b/docs/image.png differ diff --git a/monitoring/.env b/monitoring/.env new file mode 100644 index 0000000000..2a9d8985fa --- /dev/null +++ b/monitoring/.env @@ -0,0 +1 @@ +GRAFANA_ADMIN_PASSWORD=admin123 \ No newline at end of file diff --git a/monitoring/docker-compose.yml b/monitoring/docker-compose.yml new file mode 100644 index 0000000000..88f485f473 --- /dev/null +++ b/monitoring/docker-compose.yml @@ -0,0 +1,98 @@ +version: '3.8' + +services: + loki: + image: grafana/loki:3.0.0 + container_name: loki + ports: + - "3100:3100" + volumes: + - ./loki/config.yml:/etc/loki/config.yml + - loki-data:/loki + command: -config.file=/etc/loki/config.yml + networks: + - logging + labels: + logging: "promtail" + app: "loki" + healthcheck: + test: ["CMD-SHELL", "wget --no-verbose --tries=1 --spider http://localhost:3100/ready || exit 1"] + interval: 10s + timeout: 5s + retries: 5 + start_period: 10s + deploy: + resources: + limits: + cpus: '1.0' + memory: 1G + reservations: + cpus: '0.5' + memory: 512M + + promtail: + image: grafana/promtail:3.0.0 + container_name: promtail + ports: + - "9080:9080" + volumes: + - ./promtail/config.yml:/etc/promtail/config.yml + - /var/lib/docker/containers:/var/lib/docker/containers:ro + - /var/run/docker.sock:/var/run/docker.sock:ro + command: -config.file=/etc/promtail/config.yml + networks: + - logging + labels: + logging: "promtail" + app: "promtail" + depends_on: + - loki + deploy: + resources: + limits: + cpus: '0.5' + memory: 512M + reservations: + cpus: '0.25' + memory: 256M + + grafana: + image: grafana/grafana:12.3.1 + container_name: grafana + ports: + - "3000:3000" + volumes: + - grafana-data:/var/lib/grafana + environment: + - GF_AUTH_ANONYMOUS_ENABLED=false + - GF_SECURITY_ADMIN_PASSWORD=${GRAFANA_ADMIN_PASSWORD} + - GF_SECURITY_ALLOW_EMBEDDING=true + networks: + - logging + labels: + logging: "promtail" + app: "grafana" + depends_on: + - loki + healthcheck: + test: ["CMD-SHELL", "wget --no-verbose --tries=1 --spider http://localhost:3000/api/health || exit 1"] + interval: 10s + timeout: 5s + retries: 5 + start_period: 10s + deploy: + resources: + limits: + cpus: '1.0' + memory: 1G + reservations: + cpus: '0.5' + memory: 512M + +networks: + logging: + driver: bridge + +volumes: + loki-data: + grafana-data: \ No newline at end of file diff --git a/monitoring/docs/LAB07.md b/monitoring/docs/LAB07.md new file mode 100644 index 0000000000..40667eeee8 --- /dev/null +++ b/monitoring/docs/LAB07.md @@ -0,0 +1,1241 @@ +# Lab 7 β€” Observability & Logging with Loki Stack + +## 1. Architecture + +### System Overview + +``` +β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” +β”‚ Python App β”‚ β”‚ Go App β”‚ +β”‚ (port 8000) β”‚ β”‚ (port 8001) β”‚ +β”‚ JSON Logging β”‚ β”‚ JSON Logging β”‚ +β””β”€β”€β”€β”€β”€β”€β”€β”€β”¬β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β””β”€β”€β”€β”€β”€β”€β”€β”€β”¬β”€β”€β”€β”€β”€β”€β”€β”€β”˜ + β”‚ β”‚ + β”‚ Docker Logs β”‚ + β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”¬β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ + β”‚ + β”Œβ”€β”€β”€β”€β”€β”€β–Όβ”€β”€β”€β”€β”€β”€β” + β”‚ Promtail β”‚ + β”‚ (collector) β”‚ + β”‚ port 9080 β”‚ + β””β”€β”€β”€β”€β”€β”€β”¬β”€β”€β”€β”€β”€β”€β”˜ + β”‚ + β”Œβ”€β”€β”€β”€β”€β”€β–Όβ”€β”€β”€β”€β”€β”€β” + β”‚ Loki β”‚ + β”‚ (storage) β”‚ + β”‚ port 3100 β”‚ + β””β”€β”€β”€β”€β”€β”€β”¬β”€β”€β”€β”€β”€β”€β”˜ + β”‚ + β”Œβ”€β”€β”€β”€β”€β”€β–Όβ”€β”€β”€β”€β”€β”€β” + β”‚ Grafana β”‚ + β”‚ (UI) β”‚ + β”‚ port 3000 β”‚ + β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ +``` + +### Component Roles + +- **Applications**: Generate JSON-formatted logs to stdout +- **Docker**: Captures container logs and stores them in `/var/lib/docker/containers` +- **Promtail**: Discovers containers, parses JSON logs, extracts fields as labels, sends to Loki +- **Loki**: Stores logs with TSDB backend, provides query API +- **Grafana**: Visualizes logs, provides LogQL query interface + +--- + +## 2. Setup Guide + +### Prerequisites + +- Docker and Docker Compose v2 installed + +### Step-by-Step Deployment + +#### Step 1: Create Project Structure + +```bash +mkdir -p monitoring/{loki,promtail,docs} +cd monitoring +``` + +#### Step 2: Create Configuration Files + +Create `loki/config.yml`, `promtail/config.yml`, and `docker-compose.yml` (see Configuration section below). + +#### Step 3: Create Environment File + +```bash +# Create .env file for secrets +cat > .env << EOF +GRAFANA_ADMIN_PASSWORD=admin123 +EOF + +# Add to .gitignore +echo ".env" >> ../.gitignore +``` + +#### Step 4: Deploy the Stack + +```bash +docker compose up -d +``` + +#### Step 5: Verify Services + +```bash +# Check all services are running +docker compose ps + +# Test Loki +curl http://localhost:3100/ready + +# Test Promtail +curl http://localhost:9080/targets + +# Access Grafana +open http://localhost:3000 +# Login: admin / admin123 +``` + +#### Step 6: Configure Grafana Data Source + +1. Open Grafana at http://localhost:3000 +2. Login with `admin` / `admin123` +3. Navigate to **Connections** β†’ **Data sources** β†’ **Add data source** +4. Select **Loki** +5. Configure: + - URL: `http://loki:3100` + - Click **Save & Test** (should show "Data source connected") + +#### Step 7: Explore Logs + +1. Navigate to **Explore** in Grafana +2. Select **Loki** data source +3. Try query: `{app="devops-python"}` +4. You should see logs from the Python application + +--- + +## 3. Configuration + +### Loki Configuration + +**Key settings in `loki/config.yml`:** + +```yaml +auth_enabled: false # Disable multi-tenancy for single-instance + +server: + http_listen_port: 3100 + +common: + path_prefix: /loki + storage: + filesystem: + chunks_directory: /loki/chunks + rules_directory: /loki/rules + replication_factor: 1 + ring: + kvstore: + store: inmemory + +schema_config: + configs: + - from: 2024-01-01 + store: tsdb # Time Series Database (10x faster) + object_store: filesystem + schema: v13 # Latest schema for Loki 3.0+ + index: + prefix: index_ + period: 24h + +storage_config: + tsdb_shipper: + active_index_directory: /loki/tsdb-index + cache_location: /loki/tsdb-cache + +limits_config: + retention_period: 168h # 7 days + +compactor: + working_directory: /loki/compactor + compaction_interval: 10m + retention_enabled: true + retention_delete_delay: 2h + delete_request_store: filesystem +``` + +**Why these settings?** + +- **TSDB (Time Series Database)**: Up to 10x faster queries compared to previous BoltDB +- **Schema v13**: Optimized for Loki 3.0+, better compression and performance +- **7-day retention**: Balances storage costs with debugging needs +- **Compactor**: Automatically cleans up old logs based on retention policy + +### Promtail Configuration + +**Key settings in `promtail/config.yml`:** + +```yaml +server: + http_listen_port: 9080 + +clients: + - url: http://loki:3100/loki/api/v1/push + +scrape_configs: + - job_name: docker + docker_sd_configs: + - host: unix:///var/run/docker.sock + refresh_interval: 5s + filters: + - name: label + values: ["logging=promtail"] # Only containers with this label + + relabel_configs: + # Extract container name + - source_labels: ['__meta_docker_container_name'] + regex: '/(.*)' + target_label: 'container' + # Extract app label + - source_labels: ['__meta_docker_container_label_app'] + target_label: 'app' + + # Parse JSON logs at collection time + pipeline_stages: + - json: + expressions: + level: level + message: message + method: method + path: path + status_code: status_code + + # Add extracted fields as labels + - labels: + level: + method: + status_code: +``` + +**Why pipeline stages?** + +- **JSON parsing at collection time**: Extracts fields as labels for efficient querying +- **No runtime parsing overhead**: Labels are indexed during ingestion +- **Faster queries**: Can filter by `{level="ERROR"}` instead of `| json | level="ERROR"` +--- + +## 4. Application Logging + +### JSON Structured Logging Implementation + +**Python application (`app.py`):** + +```python +from pythonjsonlogger import jsonlogger +from datetime import datetime, timezone +import logging + +class CustomJsonFormatter(jsonlogger.JsonFormatter): + def add_fields(self, log_record, record, message_dict): + super().add_fields(log_record, record, message_dict) + log_record['timestamp'] = datetime.now(timezone.utc).isoformat() + log_record['level'] = record.levelname + log_record['logger'] = record.name + log_record['service'] = 'devops-python' + +# Configure logging +handler = logging.StreamHandler() +handler.setFormatter(CustomJsonFormatter('%(timestamp)s %(level)s %(name)s %(message)s')) +logger = logging.getLogger('app') +logger.addHandler(handler) +logger.setLevel(logging.INFO) +``` + +**Middleware for HTTP logging:** + +```python +@app.middleware("http") +async def log_requests(request: Request, call_next): + logger.info("Request received", extra={ + "method": request.method, + "path": str(request.url.path), + "client_ip": request.client.host, + }) + + response = await call_next(request) + + logger.info("Request completed", extra={ + "method": request.method, + "path": str(request.url.path), + "status_code": response.status_code, + "client_ip": request.client.host, + }) + + return response +``` + +**Example log output:** + +```json +{ + "timestamp": "2026-03-09T23:13:22.285158+00:00", + "level": "ERROR", + "name": "app", + "message": "HTTP exception occurred", + "status_code": 404, + "path": "/test-error-5", + "detail": "Not Found", + "logger": "app", + "service": "devops-python" +} +``` + +**Why JSON logging?** + +- **Structured data**: Easy to parse and query +- **Consistent format**: All logs have the same structure +- **Rich context**: Include method, path, status code, client IP +- **Efficient querying**: Fields can be extracted as labels in Promtail + +--- + +## 5. Dashboard + +### Panel 1: Logs Table + +**Type**: Logs visualization + +**Query**: +```logql +{app=~"devops-.*"} +``` + +**Purpose**: Shows recent logs from all applications in real-time + +**Explanation**: +- `{app=~"devops-.*"}` - Stream selector using regex to match all apps starting with "devops-" +- Displays raw log lines with timestamps +- Useful for general monitoring and debugging + +### Panel 2: Request Rate + +**Type**: Time series graph + +**Query**: +```logql +sum by (app) (rate({app=~"devops-.*"}[1m])) +``` + +**Purpose**: Visualizes logs per second by application + +**Explanation**: +- `rate({app=~"devops-.*"}[1m])` - Calculate log rate over 1-minute window +- `sum by (app)` - Aggregate by application name +- Shows traffic patterns and helps identify spikes + +### Panel 3: Error Logs + +**Type**: Logs visualization + +**Query**: +```logql +{app=~"devops-.*", level="ERROR"} +``` + +**Purpose**: Shows only ERROR level logs for quick troubleshooting + +**Explanation**: +- `level="ERROR"` - Label selector (extracted by Promtail pipeline) +- Filters logs at query time using indexed labels +- Much faster than text search `|= "ERROR"` + +### Panel 4: Log Level Distribution + +**Type**: Stat or Pie chart + +**Query**: +```logql +sum by (level) (count_over_time({app=~"devops-.*"}[5m])) +``` + +**Purpose**: Count logs by level (INFO, ERROR, etc.) over 5 minutes + +**Explanation**: +- `count_over_time({app=~"devops-.*"}[5m])` - Count logs in 5-minute window +- `sum by (level)` - Group by log level +- Helps identify error rates and log volume by severity +--- + +## 6. Production Config + +### Security Measures + +**Grafana Authentication:** +```yaml +environment: + - GF_AUTH_ANONYMOUS_ENABLED=false # Disable anonymous access + - GF_SECURITY_ADMIN_PASSWORD=${GRAFANA_ADMIN_PASSWORD} # From .env file +``` + +- Anonymous access disabled - requires login +- Admin password stored in `.env` file (not committed to git) +- Default credentials: `admin` / `admin123` + +**Docker Socket Security:** +```yaml +volumes: + - /var/run/docker.sock:/var/run/docker.sock:ro # Read-only +``` + +- Promtail has read-only access to Docker socket +- Minimizes security risk while allowing container discovery + +### Resource Limits + +**All services have resource constraints:** + +```yaml +deploy: + resources: + limits: + cpus: '1.0' + memory: 1G + reservations: + cpus: '0.5' + memory: 512M +``` + +**Service-specific limits:** +- **Loki**: 1 CPU, 1GB RAM (handles log storage and queries) +- **Promtail**: 0.5 CPU, 512MB RAM (lightweight log collector) +- **Grafana**: 1 CPU, 1GB RAM (web UI and dashboards) +- **Applications**: 0.5 CPU, 512MB RAM each + +### Health Checks + +**Loki:** +```yaml +healthcheck: + test: ["CMD-SHELL", "wget --no-verbose --tries=1 --spider http://localhost:3100/ready || exit 1"] + interval: 10s + timeout: 5s + retries: 5 + start_period: 10s +``` + +**Grafana:** +```yaml +healthcheck: + test: ["CMD-SHELL", "wget --no-verbose --tries=1 --spider http://localhost:3000/api/health || exit 1"] + interval: 10s + timeout: 5s + retries: 5 + start_period: 10s +``` + +### Retention Policy + +- **Retention period**: 7 days (168 hours) +- **Compaction interval**: 10 minutes +- **Delete delay**: 2 hours after retention expires +- Balances storage costs with debugging needs + +--- + +## 7. Testing + +### Verify Stack Deployment + +```bash +# Check all services are running and healthy +cd monitoring +docker compose ps + +# Expected output: All services "Up" and "healthy" +CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES +ffe5cbb34d59 monitoring-app-python "python app.py" 31 minutes ago Up 31 minutes 0.0.0.0:8000->8000/tcp, [::]:8000->8000/tcp app-python +f1a9c713e76b grafana/promtail:3.0.0 "/usr/bin/promtail -…" 55 minutes ago Up 27 minutes 0.0.0.0:9080->9080/tcp, [::]:9080->9080/tcp promtail +7c5c120589fd grafana/grafana:12.3.1 "/run.sh" 56 minutes ago Up 19 minutes (healthy) 0.0.0.0:3000->3000/tcp, [::]:3000->3000/tcp grafana +c75882b0895b monitoring-app-go "./myapp" About an hour ago Up About an hour 8080/tcp, 0.0.0.0:8001->8000/tcp, [::]:8001->8000/tcp app-go +562163ce9dc1 grafana/loki:3.0.0 "/usr/bin/loki -conf…" 8 hours ago Up 8 hours (healthy) 0.0.0.0:3100->3100/tcp, [::]:3100->3100/tcp loki +``` + +### Test Loki API + +```bash +# Check Loki is ready +curl http://localhost:3100/ready +# Expected: "ready" +ready + +# Query logs via API +curl -G -s "http://localhost:3100/loki/api/v1/query_range" \ + --data-urlencode 'query={app="devops-python"}' \ + --data-urlencode 'limit=5' | jq '.' + +{ + "status": "success", + "data": { + "resultType": "streams", + "result": [ + { + "stream": { + "app": "devops-python", + "container": "app-python", + "container_id": "ffe5cbb34d59e89d7fdb0dca0be0189893e8de610881c95c2be84d1b1c195f28", + "job": "docker", + "level": "INFO", + "method": "GET", + "service": "devops-python", + "service_name": "devops-python" + }, + "values": [ + [ + "1773098135528872000", + "{\"timestamp\": \"2026-03-09T23:15:35.528872+00:00\", \"level\": \"INFO\", \"name\": \"app\", \"message\": \"Incoming request\", \"method\": \"GET\", \"path\": \"/error-test-10\", \"client_ip\": \"172.18.0.1\", \"user_agent\": \"curl/8.7.1\", \"logger\": \"app\", \"service\": \"devops-python\"}" + ] + ] + }, + { + "stream": { + "app": "devops-python", + "container": "app-python", + "container_id": "ffe5cbb34d59e89d7fdb0dca0be0189893e8de610881c95c2be84d1b1c195f28", + "job": "docker", + "level": "INFO", + "method": "GET", + "service": "devops-python", + "service_name": "devops-python", + "status_code": "404" + }, + "values": [ + [ + "1773098135529437000", + "{\"timestamp\": \"2026-03-09T23:15:35.529437+00:00\", \"level\": \"INFO\", \"name\": \"app\", \"message\": \"Request completed\", \"method\": \"GET\", \"path\": \"/error-test-10\", \"status_code\": 404, \"client_ip\": \"172.18.0.1\", \"logger\": \"app\", \"service\": \"devops-python\"}" + ], + [ + "1773098135529384000", + "{\"timestamp\": \"2026-03-09T23:15:35.529384+00:00\", \"level\": \"INFO\", \"name\": \"app\", \"message\": \"Request completed\", \"method\": \"GET\", \"path\": \"/error-test-10\", \"status_code\": 404, \"client_ip\": \"172.18.0.1\", \"logger\": \"app\", \"service\": \"devops-python\"}" + ] + ] + }, + { + "stream": { + "app": "devops-python", + "container": "app-python", + "container_id": "ffe5cbb34d59e89d7fdb0dca0be0189893e8de610881c95c2be84d1b1c195f28", + "job": "docker", + "level": "ERROR", + "service": "devops-python", + "service_name": "devops-python", + "status_code": "404" + }, + "values": [ + [ + "1773098135529173000", + "{\"timestamp\": \"2026-03-09T23:15:35.529173+00:00\", \"level\": \"ERROR\", \"name\": \"app\", \"message\": \"HTTP exception occurred\", \"status_code\": 404, \"path\": \"/error-test-10\", \"detail\": \"Not Found\", \"logger\": \"app\", \"service\": \"devops-python\"}" + ], + [ + "1773098135529113000", + "{\"timestamp\": \"2026-03-09T23:15:35.529113+00:00\", \"level\": \"ERROR\", \"name\": \"app\", \"message\": \"HTTP exception occurred\", \"status_code\": 404, \"path\": \"/error-test-10\", \"detail\": \"Not Found\", \"logger\": \"app\", \"service\": \"devops-python\"}" + ] + ] + } + ], + "stats": { + "summary": { + "bytesProcessedPerSecond": 13904295, + "linesProcessedPerSecond": 55894, + "totalBytesProcessed": 78857, + "totalLinesProcessed": 317, + "execTime": 0.005671, + "queueTime": 0.000163, + "subqueries": 0, + "totalEntriesReturned": 5, + "splits": 1, + "shards": 0, + "totalPostFilterLines": 317, + "totalStructuredMetadataBytesProcessed": 4650 + }, + "querier": { + "store": { + "totalChunksRef": 0, + "totalChunksDownloaded": 0, + "chunksDownloadTime": 0, + "queryReferencedStructuredMetadata": false, + "chunk": { + "headChunkBytes": 0, + "headChunkLines": 0, + "decompressedBytes": 0, + "decompressedLines": 0, + "compressedBytes": 0, + "totalDuplicates": 0, + "postFilterLines": 0, + "headChunkStructuredMetadataBytes": 0, + "decompressedStructuredMetadataBytes": 0 + }, + "chunkRefsFetchTime": 0, + "congestionControlLatency": 0, + "pipelineWrapperFilteredLines": 0 + } + }, + "ingester": { + "totalReached": 1, + "totalChunksMatched": 6, + "totalBatches": 1, + "totalLinesSent": 5, + "store": { + "totalChunksRef": 2, + "totalChunksDownloaded": 2, + "chunksDownloadTime": 166990, + "queryReferencedStructuredMetadata": false, + "chunk": { + "headChunkBytes": 37974, + "headChunkLines": 162, + "decompressedBytes": 40883, + "decompressedLines": 155, + "compressedBytes": 4573, + "totalDuplicates": 0, + "postFilterLines": 317, + "headChunkStructuredMetadataBytes": 0, + "decompressedStructuredMetadataBytes": 4650 + }, + "chunkRefsFetchTime": 149741, + "congestionControlLatency": 0, + "pipelineWrapperFilteredLines": 0 + } + }, + "cache": { + "chunk": { + "entriesFound": 0, + "entriesRequested": 0, + "entriesStored": 0, + "bytesReceived": 0, + "bytesSent": 0, + "requests": 0, + "downloadTime": 0, + "queryLengthServed": 0 + }, + "index": { + "entriesFound": 0, + "entriesRequested": 0, + "entriesStored": 0, + "bytesReceived": 0, + "bytesSent": 0, + "requests": 0, + "downloadTime": 0, + "queryLengthServed": 0 + }, + "result": { + "entriesFound": 0, + "entriesRequested": 0, + "entriesStored": 0, + "bytesReceived": 0, + "bytesSent": 0, + "requests": 0, + "downloadTime": 0, + "queryLengthServed": 0 + }, + "statsResult": { + "entriesFound": 0, + "entriesRequested": 0, + "entriesStored": 0, + "bytesReceived": 0, + "bytesSent": 0, + "requests": 0, + "downloadTime": 0, + "queryLengthServed": 0 + }, + "volumeResult": { + "entriesFound": 0, + "entriesRequested": 0, + "entriesStored": 0, + "bytesReceived": 0, + "bytesSent": 0, + "requests": 0, + "downloadTime": 0, + "queryLengthServed": 0 + }, + "seriesResult": { + "entriesFound": 0, + "entriesRequested": 0, + "entriesStored": 0, + "bytesReceived": 0, + "bytesSent": 0, + "requests": 0, + "downloadTime": 0, + "queryLengthServed": 0 + }, + "labelResult": { + "entriesFound": 0, + "entriesRequested": 0, + "entriesStored": 0, + "bytesReceived": 0, + "bytesSent": 0, + "requests": 0, + "downloadTime": 0, + "queryLengthServed": 0 + }, + "instantMetricResult": { + "entriesFound": 0, + "entriesRequested": 0, + "entriesStored": 0, + "bytesReceived": 0, + "bytesSent": 0, + "requests": 0, + "downloadTime": 0, + "queryLengthServed": 0 + } + }, + "index": { + "totalChunks": 0, + "postFilterChunks": 0 + } + } + } +} +``` + +### Test Promtail + +```bash +# Check Promtail targets +curl http://localhost:9080/targets + +# Should show discovered containers with label "logging=promtail" + + + + + + Targets + + + + + + + + + + + + + + + + + + + + + +
+

Targets

+
+ + +
+
+ + + + + +
+

+ docker/unix:///var/run/docker.sock:80 (2/2 ready) + +

+ + + + + + + + + + + + + + + + + + + + + + + + + + +
TypeReadyLabelsDetails
+ Docker
+ 		+ + true + + + + + __address__="172.18.0.6:8000" + + __meta_docker_container_id="ffe5cbb34d59e89d7fdb0dca0be0189893e8de610881c95c2be84d1b1c195f28" + + __meta_docker_container_label_app="devops-python" + + __meta_docker_container_label_com_docker_compose_config_hash="d7517c5a68fb6bf1bf05c028fbfee6a8753bea644b611c0ab61dbb51e8109abf" + + __meta_docker_container_label_com_docker_compose_container_number="1" + + __meta_docker_container_label_com_docker_compose_depends_on="" + + __meta_docker_container_label_com_docker_compose_image="sha256:b96ea47427f533a9a5d1f84ed7b35673b94fc546497982b8a92a7256ef397521" + + __meta_docker_container_label_com_docker_compose_image_builder="classic" + + __meta_docker_container_label_com_docker_compose_oneoff="False" + + __meta_docker_container_label_com_docker_compose_project="monitoring" + + __meta_docker_container_label_com_docker_compose_project_config_files="/Users/newspec/Desktop/DevOps/DevOps-Core-Course/monitoring/docker-compose.yml" + + __meta_docker_container_label_com_docker_compose_project_working_dir="/Users/newspec/Desktop/DevOps/DevOps-Core-Course/monitoring" + + __meta_docker_container_label_com_docker_compose_replace="app-python" + + __meta_docker_container_label_com_docker_compose_service="app-python" + + __meta_docker_container_label_com_docker_compose_version="5.1.0" + + __meta_docker_container_label_logging="promtail" + + __meta_docker_container_name="/app-python" + + __meta_docker_container_network_mode="monitoring_logging" + + __meta_docker_network_id="dbbae221773ea21a02a0ec784e0d0e4cc26fb8aaaeb96c20f922fd85ec49629c" + + __meta_docker_network_ingress="false" + + __meta_docker_network_internal="false" + + __meta_docker_network_ip="172.18.0.6" + + __meta_docker_network_label_com_docker_compose_config_hash="ddec219b739fc99508f3c08de6c29964e557ed6549f4f58bb6df60e82e20dbb5" + + __meta_docker_network_label_com_docker_compose_network="logging" + + __meta_docker_network_label_com_docker_compose_project="monitoring" + + __meta_docker_network_label_com_docker_compose_version="5.1.0" + + __meta_docker_network_name="monitoring_logging" + + __meta_docker_network_scope="local" + + __meta_docker_port_private="8000" + + __meta_docker_port_public="8000" + + __meta_docker_port_public_ip="0.0.0.0" + + + + +
+ Docker
+ 		+ + true + + + + + __address__="172.18.0.5:8080" + + __meta_docker_container_id="c75882b0895b26287815c4e9e8916e0b17e476db2871b6f3c4411e2b15937ef7" + + __meta_docker_container_label_app="devops-go" + + __meta_docker_container_label_com_docker_compose_config_hash="ed019c72ac77a3d405b4a4f5b01db8d1b8a965f8f2866ac5c73d16993f7a9918" + + __meta_docker_container_label_com_docker_compose_container_number="1" + + __meta_docker_container_label_com_docker_compose_depends_on="" + + __meta_docker_container_label_com_docker_compose_image="sha256:fa3df4a039dcccba11cdd2b72d01db76094b517186e171e2c8dfea2a1bd469c4" + + __meta_docker_container_label_com_docker_compose_image_builder="classic" + + __meta_docker_container_label_com_docker_compose_oneoff="False" + + __meta_docker_container_label_com_docker_compose_project="monitoring" + + __meta_docker_container_label_com_docker_compose_project_config_files="/Users/newspec/Desktop/DevOps/DevOps-Core-Course/monitoring/docker-compose.yml" + + __meta_docker_container_label_com_docker_compose_project_working_dir="/Users/newspec/Desktop/DevOps/DevOps-Core-Course/monitoring" + + __meta_docker_container_label_com_docker_compose_service="app-go" + + __meta_docker_container_label_com_docker_compose_version="5.1.0" + + __meta_docker_container_label_logging="promtail" + + __meta_docker_container_name="/app-go" + + __meta_docker_container_network_mode="monitoring_logging" + + __meta_docker_network_id="dbbae221773ea21a02a0ec784e0d0e4cc26fb8aaaeb96c20f922fd85ec49629c" + + __meta_docker_network_ingress="false" + + __meta_docker_network_internal="false" + + __meta_docker_network_ip="172.18.0.5" + + __meta_docker_network_label_com_docker_compose_config_hash="ddec219b739fc99508f3c08de6c29964e557ed6549f4f58bb6df60e82e20dbb5" + + __meta_docker_network_label_com_docker_compose_network="logging" + + __meta_docker_network_label_com_docker_compose_project="monitoring" + + __meta_docker_network_label_com_docker_compose_version="5.1.0" + + __meta_docker_network_name="monitoring_logging" + + __meta_docker_network_scope="local" + + __meta_docker_port_private="8080" + + + + +
+
+ +
+ + + +``` + +### Generate Test Traffic + +```bash +# Generate successful requests +for i in {1..20}; do + curl http://localhost:8000/ + curl http://localhost:8000/health +done + +# Generate error requests +for i in {1..10}; do + curl http://localhost:8000/nonexistent-$i +done +``` + +### Verify Logs in Grafana + +1. Open Grafana: http://localhost:3000 +2. Login: `*****` +3. Navigate to **Explore** +4. Select **Loki** data source +5. Try these queries: + +```logql +# All logs +{app="devops-python"} + +# Only errors +{app="devops-python", level="ERROR"} + +# Only INFO logs +{app="devops-python", level="INFO"} + +# Count by level +sum by (level) (count_over_time({app="devops-python"}[5m])) +``` + +### Test LogQL Queries + +**Basic filtering:** +```bash +# All logs from Python app +{app="devops-python"} + +# Logs from both apps +{app=~"devops-.*"} + +# Only ERROR level +{app="devops-python", level="ERROR"} + +# Specific HTTP method +{app="devops-python", method="GET"} +``` + +**Metrics from logs:** +```bash +# Request rate +rate({app="devops-python"}[1m]) + +# Count by level +sum by (level) (count_over_time({app="devops-python"}[5m])) + +# Error rate +sum(rate({app="devops-python", level="ERROR"}[5m])) +``` + +--- + +## 8. Challenges + +### Challenge 1: Loki Configuration Errors + +**Problem**: Initial Loki startup failed with deprecated configuration fields. + +**Error message**: +``` +error parsing config: yaml: unmarshal errors: + line X: field max_look_back_period not found +``` + +**Solution**: +- Removed deprecated `max_look_back_period` from `chunk_store_config` +- Added `delete_request_store: filesystem` to compactor configuration +- Updated to TSDB-specific configuration for Loki 3.0 + +### Challenge 2: Promtail Not Collecting Logs + +**Problem**: Promtail wasn't discovering containers or collecting logs. + +**Root cause**: Missing Docker socket and container log directory mounts. + +**Solution**: +```yaml +volumes: + - /var/run/docker.sock:/var/run/docker.sock:ro + - /var/lib/docker/containers:/var/lib/docker/containers:ro +``` + +**Additional fix**: Added label-based filtering to only collect from containers with `logging=promtail` label. + +### Challenge 3: Mixed Log Formats + +**Problem**: Application logs contained both JSON and plain text (from uvicorn), preventing consistent parsing. + +**Example**: +``` +INFO: 127.0.0.1:52134 - "GET / HTTP/1.1" 200 OK +{"timestamp": "2026-03-09T22:30:25.607856+00:00", "level": "INFO", ...} +``` + +**Solution**: +- Disabled uvicorn access logs: `uvicorn.run(app, access_log=False, log_config=None)` +- Ensured all application logs use JSON formatter +- Result: Pure JSON output for consistent parsing + +### Challenge 4: JSON Parsing in LogQL + +**Problem**: Query `{app="devops-python"} | json | level="ERROR"` returned "No data" despite ERROR logs existing. + +**Root cause**: Docker logs were already JSON-encoded, causing Loki to store them as escaped strings: +```json +"{\"timestamp\": \"...\", \"level\": \"ERROR\", ...}" +``` + +The `| json` parser couldn't extract fields from escaped JSON. + +**Solution**: Added **pipeline stages** to Promtail configuration to parse JSON at collection time: + +```yaml +pipeline_stages: + - json: + expressions: + level: level + method: method + status_code: status_code + - labels: + level: + method: + status_code: +``` + +**Result**: Fields are now extracted as labels during ingestion, enabling direct filtering: +- `{level="ERROR"}` instead of `| json | level="ERROR"` +- Faster queries (no runtime parsing) +- Fields indexed at ingestion time +--- + +## Evidence of Completion + +### Task 1: Deploy Loki Stack + +**Screenshot showing logs from at least 3 containers in Grafana Explore:** +![alt text](image.png) +![alt text](image-1.png) +![alt text](image-2.png) +![alt text](image-3.png) +![alt text](image-4.png) + +### Task 2: Integrate Your Applications + +**Screenshot of JSON log output from your app:** +![alt text](image-5.png) +**Screenshot of Grafana showing logs from both applications:** +![alt text](image-7.png) +![alt text](image-8.png) +**At least 3 different LogQL queries that work:** +- `{app="devops-python"}` +![alt text](image-9.png) +- `{app="devops-python"} |= "ERROR"` +![alt text](image-10.png) +- `{app="devops-python"} | json | method="GET"` +![alt text](image-11.png) +### Task 3: Build Log Dashboard + +**Screenshot of your dashboard showing all 4 panels with real data.:** +![alt text](image-12.png) + +### Task 4: Production Readiness + +**docker-compose ps showing all services healthy*** +``` +docker ps +CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES +9a3014c256e9 monitoring-app-go "./myapp" 11 minutes ago Up 11 minutes 0.0.0.0:8001->8080/tcp, [::]:8001->8080/tcp app-go +41601c3d6499 grafana/promtail:3.0.0 "/usr/bin/promtail -…" 12 minutes ago Up 12 minutes 0.0.0.0:9080->9080/tcp, [::]:9080->9080/tcp promtail +b68a6c76cc9a grafana/grafana:12.3.1 "/run.sh" 12 minutes ago Up 12 minutes (healthy) 0.0.0.0:3000->3000/tcp, [::]:3000->3000/tcp grafana +2b62dd0622f8 grafana/loki:3.0.0 "/usr/bin/loki -conf…" 12 minutes ago Up 12 minutes (healthy) 0.0.0.0:3100->3100/tcp, [::]:3100->3100/tcp loki +ffe5cbb34d59 monitoring-app-python "python app.py" 57 minutes ago Up 57 minutes 0.0.0.0:8000->8000/tcp, [::]:8000->8000/tcp app-python +``` +**Screenshot of Grafana login page (no anonymous access):** +![alt text](image-13.png) + +### Bonus β€” Ansible Automation +**Ansible playbook execution output:** +```bash +Using /Users/newspec/Desktop/DevOps/DevOps-Core-Course/ansible/ansible.cfg as config file + +PLAY [Deploy Monitoring Stack] ********************************************************************************************************************************** + +TASK [Gathering Facts] ****************************************************************************************************************************************** +ok: [localhost] + +TASK [monitoring : Include setup tasks] ************************************************************************************************************************* +included: /Users/newspec/Desktop/DevOps/DevOps-Core-Course/ansible/roles/monitoring/tasks/setup.yml for localhost + +TASK [monitoring : Create monitoring directory structure] ******************************************************************************************************* +ok: [localhost] => (item=/Users/newspec/Desktop/DevOps/DevOps-Core-Course/ansible/../monitoring) => {"ansible_loop_var": "item", "changed": false, "gid": 20, "group": "staff", "item": "/Users/newspec/Desktop/DevOps/DevOps-Core-Course/ansible/../monitoring", "mode": "0755", "owner": "newspec", "path": "/Users/newspec/Desktop/DevOps/DevOps-Core-Course/ansible/../monitoring", "size": 224, "state": "directory", "uid": 501} +ok: [localhost] => (item=/Users/newspec/Desktop/DevOps/DevOps-Core-Course/ansible/../monitoring/loki) => {"ansible_loop_var": "item", "changed": false, "gid": 20, "group": "staff", "item": "/Users/newspec/Desktop/DevOps/DevOps-Core-Course/ansible/../monitoring/loki", "mode": "0755", "owner": "newspec", "path": "/Users/newspec/Desktop/DevOps/DevOps-Core-Course/ansible/../monitoring/loki", "size": 96, "state": "directory", "uid": 501} +ok: [localhost] => (item=/Users/newspec/Desktop/DevOps/DevOps-Core-Course/ansible/../monitoring/promtail) => {"ansible_loop_var": "item", "changed": false, "gid": 20, "group": "staff", "item": "/Users/newspec/Desktop/DevOps/DevOps-Core-Course/ansible/../monitoring/promtail", "mode": "0755", "owner": "newspec", "path": "/Users/newspec/Desktop/DevOps/DevOps-Core-Course/ansible/../monitoring/promtail", "size": 96, "state": "directory", "uid": 501} +ok: [localhost] => (item=/Users/newspec/Desktop/DevOps/DevOps-Core-Course/ansible/../monitoring/docs) => {"ansible_loop_var": "item", "changed": false, "gid": 20, "group": "staff", "item": "/Users/newspec/Desktop/DevOps/DevOps-Core-Course/ansible/../monitoring/docs", "mode": "0755", "owner": "newspec", "path": "/Users/newspec/Desktop/DevOps/DevOps-Core-Course/ansible/../monitoring/docs", "size": 544, "state": "directory", "uid": 501} + +TASK [monitoring : Template Loki configuration] ***************************************************************************************************************** +ok: [localhost] => {"changed": false, "checksum": "d42de2d0cd64379828e0bf9003a88aeceff2f0b1", "dest": "/Users/newspec/Desktop/DevOps/DevOps-Core-Course/ansible/../monitoring/loki/config.yml", "gid": 20, "group": "staff", "mode": "0644", "owner": "newspec", "path": "/Users/newspec/Desktop/DevOps/DevOps-Core-Course/ansible/../monitoring/loki/config.yml", "size": 1457, "state": "file", "uid": 501} + +TASK [monitoring : Template Promtail configuration] ************************************************************************************************************* +ok: [localhost] => {"changed": false, "checksum": "af82481cc89df3f966895d245c9433a2e0c2e411", "dest": "/Users/newspec/Desktop/DevOps/DevOps-Core-Course/ansible/../monitoring/promtail/config.yml", "gid": 20, "group": "staff", "mode": "0644", "owner": "newspec", "path": "/Users/newspec/Desktop/DevOps/DevOps-Core-Course/ansible/../monitoring/promtail/config.yml", "size": 1731, "state": "file", "uid": 501} + +TASK [monitoring : Template Docker Compose file] **************************************************************************************************************** +changed: [localhost] => {"changed": true, "checksum": "511a7e9611d7defd5ab4b7d1a5549b1c4d6956de", "dest": "/Users/newspec/Desktop/DevOps/DevOps-Core-Course/ansible/../monitoring/docker-compose.yml", "gid": 20, "group": "staff", "md5sum": "bb3a3b54449ce0caed008059ffc5f4ed", "mode": "0644", "owner": "newspec", "size": 2185, "src": "/Users/newspec/.ansible/tmp/ansible-tmp-1773101608.078568-37435-183065845996472/.source.yml", "state": "file", "uid": 501} + +TASK [monitoring : Create .env file for secrets] **************************************************************************************************************** +changed: [localhost] => {"censored": "the output has been hidden due to the fact that 'no_log: true' was specified for this result", "changed": true} + +TASK [monitoring : Include deployment tasks] ******************************************************************************************************************** +included: /Users/newspec/Desktop/DevOps/DevOps-Core-Course/ansible/roles/monitoring/tasks/deploy.yml for localhost + +TASK [monitoring : Deploy monitoring stack with Docker Compose] ************************************************************************************************* +[ERROR]: Task failed: Module failed: failed to connect to the docker API at unix:///var/run/docker.sock; check if the path is correct and if the daemon is running: dial unix /var/run/docker.sock: connect: no such file or directory +Origin: /Users/newspec/Desktop/DevOps/DevOps-Core-Course/ansible/roles/monitoring/tasks/deploy.yml:4:3 + +2 # Deployment tasks for monitoring stack +3 +4 - name: Deploy monitoring stack with Docker Compose + ^ column 3 + +fatal: [localhost]: FAILED! => {"changed": false, "cmd": "/opt/homebrew/bin/docker --host unix:///var/run/docker.sock version --format '{{ json . }}'", "msg": "failed to connect to the docker API at unix:///var/run/docker.sock; check if the path is correct and if the daemon is running: dial unix /var/run/docker.sock: connect: no such file or directory", "rc": 1, "stderr": "failed to connect to the docker API at unix:///var/run/docker.sock; check if the path is correct and if the daemon is running: dial unix /var/run/docker.sock: connect: no such file or directory\n", "stderr_lines": ["failed to connect to the docker API at unix:///var/run/docker.sock; check if the path is correct and if the daemon is running: dial unix /var/run/docker.sock: connect: no such file or directory"], "stdout": "{\"Client\":{\"Platform\":{\"Name\":\"Docker Engine - Community\"},\"Version\":\"29.3.0\",\"ApiVersion\":\"1.54\",\"DefaultAPIVersion\":\"1.54\",\"GitCommit\":\"5927d80c76\",\"GoVersion\":\"go1.26.1\",\"Os\":\"darwin\",\"Arch\":\"arm64\",\"BuildTime\":\"Thu Mar 5 14:22:32 2026\",\"Context\":\"default\"},\"Server\":null}\n", "stdout_lines": ["{\"Client\":{\"Platform\":{\"Name\":\"Docker Engine - Community\"},\"Version\":\"29.3.0\",\"ApiVersion\":\"1.54\",\"DefaultAPIVersion\":\"1.54\",\"GitCommit\":\"5927d80c76\",\"GoVersion\":\"go1.26.1\",\"Os\":\"darwin\",\"Arch\":\"arm64\",\"BuildTime\":\"Thu Mar 5 14:22:32 2026\",\"Context\":\"default\"},\"Server\":null}"]} + +PLAY RECAP ****************************************************************************************************************************************************** +localhost : ok=8 changed=2 unreachable=0 failed=1 skipped=0 rescued=0 ignored=0 + +newspec@172 ansible % +``` +**Idempotency test (run twice, second shows no changes):** +```bash +LAY [Deploy Monitoring Stack] ************************************************* + +TASK [Gathering Facts] ********************************************************* +ok: [localhost] + +TASK [monitoring : Include setup tasks] **************************************** +included: /Users/newspec/Desktop/DevOps/DevOps-Core-Course/ansible/roles/monitoring/tasks/setup.yml for localhost + +TASK [monitoring : Create monitoring directory structure] ********************** +ok: [localhost] => (item=/Users/newspec/Desktop/DevOps/DevOps-Core-Course/ansible/../monitoring) +ok: [localhost] => (item=/Users/newspec/Desktop/DevOps/DevOps-Core-Course/ansible/../monitoring/loki) +ok: [localhost] => (item=/Users/newspec/Desktop/DevOps/DevOps-Core-Course/ansible/../monitoring/promtail) +ok: [localhost] => (item=/Users/newspec/Desktop/DevOps/DevOps-Core-Course/ansible/../monitoring/docs) + +TASK [monitoring : Template Loki configuration] ******************************** +ok: [localhost] + +TASK [monitoring : Template Promtail configuration] **************************** +ok: [localhost] + +TASK [monitoring : Template Docker Compose file] ******************************* +ok: [localhost] + +TASK [monitoring : Create .env file for secrets] ******************************* +ok: [localhost] + +TASK [Display access information] ********************************************** +ok: [localhost] => { + "msg": "========================================\nMonitoring Stack Deployed Successfully!\n========================================\n\nGrafana UI: http://localhost:3000\nUsername: admin\nPassword: admin123\n\nLoki API: http://localhost:3100\n\nNext Steps:\n1. Open Grafana in your browser\n2. Add Loki data source (http://loki:3100)\n3. Explore logs in the Explore tab\n4. Create dashboards for your applications\n\n========================================" +} + +PLAY RECAP ********************************************************************* +localhost : ok=8 changed=0 unreachable=0 failed=0 skipped=0 rescued=0 ignored=0 + + +=== Idempotency Check === +localhost : ok=8 changed=0 unreachable=0 failed=0 skipped=0 rescued=0 ignored=0 +``` \ No newline at end of file diff --git a/monitoring/docs/image-1.png b/monitoring/docs/image-1.png new file mode 100644 index 0000000000..0cbcdaa411 Binary files /dev/null and b/monitoring/docs/image-1.png differ diff --git a/monitoring/docs/image-10.png b/monitoring/docs/image-10.png new file mode 100644 index 0000000000..f2347ba966 Binary files /dev/null and b/monitoring/docs/image-10.png differ diff --git a/monitoring/docs/image-11.png b/monitoring/docs/image-11.png new file mode 100644 index 0000000000..3657509025 Binary files /dev/null and b/monitoring/docs/image-11.png differ diff --git a/monitoring/docs/image-12.png b/monitoring/docs/image-12.png new file mode 100644 index 0000000000..f9fa917fc6 Binary files /dev/null and b/monitoring/docs/image-12.png differ diff --git a/monitoring/docs/image-13.png b/monitoring/docs/image-13.png new file mode 100644 index 0000000000..8d4bd0ea5a Binary files /dev/null and b/monitoring/docs/image-13.png differ diff --git a/monitoring/docs/image-2.png b/monitoring/docs/image-2.png new file mode 100644 index 0000000000..e4868d2099 Binary files /dev/null and b/monitoring/docs/image-2.png differ diff --git a/monitoring/docs/image-3.png b/monitoring/docs/image-3.png new file mode 100644 index 0000000000..e539c1f7f0 Binary files /dev/null and b/monitoring/docs/image-3.png differ diff --git a/monitoring/docs/image-4.png b/monitoring/docs/image-4.png new file mode 100644 index 0000000000..d11e230e0a Binary files /dev/null and b/monitoring/docs/image-4.png differ diff --git a/monitoring/docs/image-5.png b/monitoring/docs/image-5.png new file mode 100644 index 0000000000..9eab0bbe58 Binary files /dev/null and b/monitoring/docs/image-5.png differ diff --git a/monitoring/docs/image-6.png b/monitoring/docs/image-6.png new file mode 100644 index 0000000000..911ebcd52f Binary files /dev/null and b/monitoring/docs/image-6.png differ diff --git a/monitoring/docs/image-7.png b/monitoring/docs/image-7.png new file mode 100644 index 0000000000..c31195b8ac Binary files /dev/null and b/monitoring/docs/image-7.png differ diff --git a/monitoring/docs/image-8.png b/monitoring/docs/image-8.png new file mode 100644 index 0000000000..3e8a73b178 Binary files /dev/null and b/monitoring/docs/image-8.png differ diff --git a/monitoring/docs/image-9.png b/monitoring/docs/image-9.png new file mode 100644 index 0000000000..0fec9e58bf Binary files /dev/null and b/monitoring/docs/image-9.png differ diff --git a/monitoring/docs/image.png b/monitoring/docs/image.png new file mode 100644 index 0000000000..0cfcd7dddd Binary files /dev/null and b/monitoring/docs/image.png differ diff --git a/monitoring/loki/config.yml b/monitoring/loki/config.yml new file mode 100644 index 0000000000..2ff6d38004 --- /dev/null +++ b/monitoring/loki/config.yml @@ -0,0 +1,75 @@ +auth_enabled: false + +server: + http_listen_port: 3100 + grpc_listen_port: 9096 + +common: + instance_addr: 127.0.0.1 + path_prefix: /loki + storage: + filesystem: + chunks_directory: /loki/chunks + rules_directory: /loki/rules + replication_factor: 1 + ring: + kvstore: + store: inmemory + +query_range: + results_cache: + cache: + embedded_cache: + enabled: true + max_size_mb: 100 + +schema_config: + configs: + - from: 2024-01-01 + store: tsdb + object_store: filesystem + schema: v13 + index: + prefix: index_ + period: 24h + +storage_config: + tsdb_shipper: + active_index_directory: /loki/tsdb-index + cache_location: /loki/tsdb-cache + cache_ttl: 24h + filesystem: + directory: /loki/chunks + +limits_config: + retention_period: 168h + reject_old_samples: true + reject_old_samples_max_age: 168h + ingestion_rate_mb: 10 + ingestion_burst_size_mb: 20 + max_query_series: 500 + max_query_parallelism: 32 + +compactor: + working_directory: /loki/compactor + compaction_interval: 10m + retention_enabled: true + retention_delete_delay: 2h + retention_delete_worker_count: 150 + delete_request_store: filesystem + +table_manager: + retention_deletes_enabled: true + retention_period: 168h + +ruler: + storage: + type: local + local: + directory: /loki/rules + rule_path: /loki/rules-temp + alertmanager_url: http://localhost:9093 + ring: + kvstore: + store: inmemory + enable_api: true \ No newline at end of file diff --git a/monitoring/promtail/config.yml b/monitoring/promtail/config.yml new file mode 100644 index 0000000000..170aca60f3 --- /dev/null +++ b/monitoring/promtail/config.yml @@ -0,0 +1,63 @@ +server: + http_listen_port: 9080 + grpc_listen_port: 0 + +positions: + filename: /tmp/positions.yaml + +clients: + - url: http://loki:3100/loki/api/v1/push + +scrape_configs: + - job_name: docker + docker_sd_configs: + - host: unix:///var/run/docker.sock + refresh_interval: 5s + filters: + - name: label + values: ["logging=promtail"] + relabel_configs: + # Extract container name + - source_labels: ['__meta_docker_container_name'] + regex: '/(.*)' + target_label: 'container' + # Extract container ID + - source_labels: ['__meta_docker_container_id'] + target_label: 'container_id' + # Extract image name + - source_labels: ['__meta_docker_container_label_com_docker_compose_service'] + target_label: 'service' + # Extract app label if present + - source_labels: ['__meta_docker_container_label_app'] + target_label: 'app' + # Add job label + - source_labels: ['__meta_docker_container_label_logging'] + target_label: 'job' + replacement: 'docker' + + # Pipeline stages to parse JSON logs + pipeline_stages: + # Parse JSON from log line + - json: + expressions: + level: level + message: message + timestamp: timestamp + logger: logger + service: service + method: method + path: path + status_code: status_code + client_ip: client_ip + + # Add extracted fields as labels + - labels: + level: + service: + method: + status_code: + + # Use timestamp from log if available + - timestamp: + source: timestamp + format: RFC3339Nano \ No newline at end of file diff --git a/pulumi/Pulumi.yaml b/pulumi/Pulumi.yaml new file mode 100644 index 0000000000..0e9b5069df --- /dev/null +++ b/pulumi/Pulumi.yaml @@ -0,0 +1,3 @@ +name: lab04-pulumi +runtime: python +description: Lab 04 - Infrastructure as Code with Pulumi (Yandex Cloud) \ No newline at end of file diff --git a/pulumi/__main__.py b/pulumi/__main__.py new file mode 100644 index 0000000000..e719228a87 --- /dev/null +++ b/pulumi/__main__.py @@ -0,0 +1,150 @@ +""" +Lab 04 - Pulumi Infrastructure as Code +Provisions a VM on Yandex Cloud with network and security configuration +""" + +import pulumi +import pulumi_yandex as yandex + +# Get configuration +config = pulumi.Config() +folder_id = config.require("folder_id") +zone = config.get("zone") or "ru-central1-a" +vm_name = config.get("vm_name") or "lab04-pulumi-vm" +ssh_user = config.get("ssh_user") or "ubuntu" +ssh_public_key_path = config.get("ssh_public_key_path") or "~/.ssh/id_rsa.pub" +# CIDR allowed to SSH, e.g. 203.0.113.10/32 +ssh_allowed_cidr = config.require("ssh_allowed_cidr") + +# Read SSH public key +ssh_key_expanded = ssh_public_key_path.replace( + "~", pulumi.runtime.get_config("HOME") or "~" +) +with open(ssh_key_expanded, "r") as f: + ssh_public_key = f.read().strip() + +# Get latest Ubuntu 24.04 image +ubuntu_image = yandex.get_compute_image( + family="ubuntu-2404-lts", + folder_id="standard-images" +) + +# Create VPC network +network = yandex.VpcNetwork( + "lab04-network", + name="lab04-pulumi-network", + description="Network for Lab 04 Pulumi VM", + folder_id=folder_id +) + +# Create subnet +subnet = yandex.VpcSubnet( + "lab04-subnet", + name="lab04-pulumi-subnet", + description="Subnet for Lab 04 Pulumi VM", + v4_cidr_blocks=["10.128.0.0/24"], + zone=zone, + network_id=network.id, + folder_id=folder_id +) + +# Create security group +security_group = yandex.VpcSecurityGroup( + "lab04-sg", + name="lab04-pulumi-security-group", + description="Security group for Lab 04 Pulumi VM", + network_id=network.id, + folder_id=folder_id, + ingresses=[ + # Allow SSH from specific IP + yandex.VpcSecurityGroupIngressArgs( + protocol="TCP", + description="Allow SSH from my IP", + v4_cidr_blocks=[ssh_allowed_cidr], + port=22 + ), + # Allow HTTP + yandex.VpcSecurityGroupIngressArgs( + protocol="TCP", + description="Allow HTTP", + v4_cidr_blocks=["0.0.0.0/0"], + port=80 + ), + # Allow custom port 5000 + yandex.VpcSecurityGroupIngressArgs( + protocol="TCP", + description="Allow app port 5000", + v4_cidr_blocks=["0.0.0.0/0"], + port=5000 + ), + ], + egresses=[ + # Allow all outbound traffic + yandex.VpcSecurityGroupEgressArgs( + protocol="ANY", + description="Allow all outbound traffic", + v4_cidr_blocks=["0.0.0.0/0"], + from_port=0, + to_port=65535 + ), + ] +) + +# Create VM instance +vm = yandex.ComputeInstance( + "lab04-vm", + name=vm_name, + hostname=vm_name, + platform_id="standard-v2", + zone=zone, + folder_id=folder_id, + resources=yandex.ComputeInstanceResourcesArgs( + cores=2, + memory=1, + core_fraction=20 # Free tier: 20% CPU + ), + boot_disk=yandex.ComputeInstanceBootDiskArgs( + initialize_params=yandex.ComputeInstanceBootDiskInitializeParamsArgs( + image_id=ubuntu_image.id, + size=10, # 10 GB + type="network-hdd" + ) + ), + network_interfaces=[ + yandex.ComputeInstanceNetworkInterfaceArgs( + subnet_id=subnet.id, + nat=True, # Assign public IP + security_group_ids=[security_group.id] + ) + ], + metadata={ + "ssh-keys": f"{ssh_user}:{ssh_public_key}" + }, + labels={ + "environment": "lab04", + "managed_by": "pulumi", + "purpose": "devops-course" + }, + scheduling_policy=yandex.ComputeInstanceSchedulingPolicyArgs( + preemptible=False + ) +) + +# Export outputs +pulumi.export("vm_id", vm.id) +pulumi.export("vm_name", vm.name) +pulumi.export("vm_public_ip", vm.network_interfaces[0].nat_ip_address) +pulumi.export("vm_private_ip", vm.network_interfaces[0].ip_address) +pulumi.export("network_id", network.id) +pulumi.export("subnet_id", subnet.id) +pulumi.export("ssh_command", vm.network_interfaces[0].nat_ip_address.apply( + lambda ip: f"ssh {ssh_user}@{ip}" +)) +pulumi.export("connection_info", { + "public_ip": vm.network_interfaces[0].nat_ip_address, + "private_ip": vm.network_interfaces[0].ip_address, + "ssh_user": ssh_user, + "ssh_command": vm.network_interfaces[0].nat_ip_address.apply( + lambda ip: f"ssh {ssh_user}@{ip}" + ) +}) diff --git a/pulumi/requirements.txt b/pulumi/requirements.txt new file mode 100644 index 0000000000..2356228903 --- /dev/null +++ b/pulumi/requirements.txt @@ -0,0 +1,2 @@ +pulumi>=3.0.0,<4.0.0 +pulumi-yandex>=0.13.0 \ No newline at end of file diff --git a/terraform/.terraformrc b/terraform/.terraformrc new file mode 100644 index 0000000000..9bc7728211 --- /dev/null +++ b/terraform/.terraformrc @@ -0,0 +1,9 @@ +provider_installation { + network_mirror { + url = "https://terraform-mirror.yandexcloud.net/" + include = ["registry.terraform.io/*/*"] + } + direct { + exclude = ["registry.terraform.io/*/*"] + } +} \ No newline at end of file diff --git a/terraform/.tflint.hcl b/terraform/.tflint.hcl new file mode 100644 index 0000000000..96e00d361b --- /dev/null +++ b/terraform/.tflint.hcl @@ -0,0 +1,24 @@ +plugin "terraform" { + enabled = true + preset = "recommended" +} + +rule "terraform_naming_convention" { + enabled = true +} + +rule "terraform_documented_variables" { + enabled = true +} + +rule "terraform_documented_outputs" { + enabled = true +} + +rule "terraform_unused_declarations" { + enabled = true +} + +rule "terraform_deprecated_index" { + enabled = true +} \ No newline at end of file diff --git a/terraform/github.tf b/terraform/github.tf new file mode 100644 index 0000000000..39b5a40c02 --- /dev/null +++ b/terraform/github.tf @@ -0,0 +1,52 @@ +# GitHub Provider Configuration for Repository Management +# This file demonstrates importing existing infrastructure into Terraform +# Note: required_providers for github is defined in main.tf + +provider "github" { + token = var.github_token + owner = var.github_owner +} + +# Import existing DevOps-Core-Course repository +resource "github_repository" "devops_course" { + name = "DevOps-Core-Course" + description = "DevOps Engineering: Core Practices - Lab assignments and projects" + visibility = "public" + + has_issues = true + has_wiki = false + has_projects = false + has_downloads = true + + allow_merge_commit = true + allow_squash_merge = true + allow_rebase_merge = true + allow_auto_merge = false + + delete_branch_on_merge = true + + topics = [ + "devops", + "terraform", + "pulumi", + "docker", + "kubernetes", + "ansible", + "ci-cd", + "infrastructure-as-code" + ] +} + +# Branch protection for master branch (optional) +resource "github_branch_protection" "master_protection" { + repository_id = github_repository.devops_course.node_id + pattern = "master" + + required_pull_request_reviews { + dismiss_stale_reviews = true + require_code_owner_reviews = false + required_approving_review_count = 0 + } + + enforce_admins = false +} \ No newline at end of file diff --git a/terraform/main.tf b/terraform/main.tf new file mode 100644 index 0000000000..1b24dfba9f --- /dev/null +++ b/terraform/main.tf @@ -0,0 +1,118 @@ +terraform { + required_providers { + yandex = { + source = "yandex-cloud/yandex" + version = "~> 0.187" + } + github = { + source = "integrations/github" + version = "~> 5.0" + } + } + required_version = ">= 1.9.0" +} + +provider "yandex" { + service_account_key_file = pathexpand(var.service_account_key_file) + cloud_id = var.cloud_id + folder_id = var.folder_id + zone = var.zone +} + +# Get latest Ubuntu 24.04 image +data "yandex_compute_image" "ubuntu" { + family = "ubuntu-2404-lts" +} + +# Create VPC network +resource "yandex_vpc_network" "lab04_network" { + name = "lab04-network" + description = "Network for Lab 04 VM" +} + +# Create subnet +resource "yandex_vpc_subnet" "lab04_subnet" { + name = "lab04-subnet" + description = "Subnet for Lab 04 VM" + v4_cidr_blocks = ["10.128.0.0/24"] + zone = var.zone + network_id = yandex_vpc_network.lab04_network.id +} + +# Create security group with required rules +resource "yandex_vpc_security_group" "lab04_sg" { + name = "lab04-sg" + description = "Lab04 security group" + network_id = yandex_vpc_network.lab04_network.id + + ingress { + protocol = "TCP" + description = "SSH from my IP" + v4_cidr_blocks = [var.ssh_allowed_cidr] + port = 22 + } + + ingress { + protocol = "TCP" + description = "HTTP" + v4_cidr_blocks = ["0.0.0.0/0"] + port = 80 + } + + ingress { + protocol = "TCP" + description = "App 5000" + v4_cidr_blocks = ["0.0.0.0/0"] + port = 5000 + } + + egress { + protocol = "ANY" + description = "Allow all egress" + v4_cidr_blocks = ["0.0.0.0/0"] + from_port = 0 + to_port = 65535 + } +} + +# Create VM instance +resource "yandex_compute_instance" "lab04_vm" { + name = var.vm_name + hostname = var.vm_name + platform_id = "standard-v2" + zone = var.zone + + resources { + cores = 2 + memory = 1 + core_fraction = 20 # Free tier: 20% CPU + } + + boot_disk { + initialize_params { + image_id = data.yandex_compute_image.ubuntu.id + size = 10 # 10 GB HDD + type = "network-hdd" + } + } + + network_interface { + subnet_id = yandex_vpc_subnet.lab04_subnet.id + nat = true # Assign public IP + security_group_ids = [yandex_vpc_security_group.lab04_sg.id] + } + + metadata = { + ssh-keys = "${var.ssh_user}:${file(var.ssh_public_key_path)}" + } + + labels = { + environment = "lab04" + managed_by = "terraform" + purpose = "devops-course" + } + + scheduling_policy { + preemptible = false + } +} \ No newline at end of file diff --git a/terraform/outputs.tf b/terraform/outputs.tf new file mode 100644 index 0000000000..0699bacaae --- /dev/null +++ b/terraform/outputs.tf @@ -0,0 +1,53 @@ +# VM outputs +output "vm_id" { + description = "ID of the created VM" + value = yandex_compute_instance.lab04_vm.id +} + +output "vm_name" { + description = "Name of the created VM" + value = yandex_compute_instance.lab04_vm.name +} + +output "vm_public_ip" { + description = "Public IP address of the VM" + value = yandex_compute_instance.lab04_vm.network_interface[0].nat_ip_address +} + +output "vm_private_ip" { + description = "Private IP address of the VM" + value = yandex_compute_instance.lab04_vm.network_interface[0].ip_address +} + +# Network outputs +output "network_id" { + description = "ID of the VPC network" + value = yandex_vpc_network.lab04_network.id +} + +output "subnet_id" { + description = "ID of the subnet" + value = yandex_vpc_subnet.lab04_subnet.id +} + +output "security_group_id" { + description = "ID of the security group" + value = yandex_vpc_security_group.lab04_sg.id +} + +# SSH connection command +output "ssh_command" { + description = "SSH command to connect to the VM" + value = "ssh ${var.ssh_user}@${yandex_compute_instance.lab04_vm.network_interface[0].nat_ip_address}" +} + +# Connection info +output "connection_info" { + description = "Complete connection information" + value = { + public_ip = yandex_compute_instance.lab04_vm.network_interface[0].nat_ip_address + private_ip = yandex_compute_instance.lab04_vm.network_interface[0].ip_address + ssh_user = var.ssh_user + ssh_command = "ssh ${var.ssh_user}@${yandex_compute_instance.lab04_vm.network_interface[0].nat_ip_address}" + } +} \ No newline at end of file diff --git a/terraform/variables.tf b/terraform/variables.tf new file mode 100644 index 0000000000..5efc57b2e9 --- /dev/null +++ b/terraform/variables.tf @@ -0,0 +1,61 @@ +# Yandex Cloud configuration +variable "cloud_id" { + description = "Yandex Cloud ID" + type = string +} + +variable "folder_id" { + description = "Yandex Cloud folder ID" + type = string +} + +variable "service_account_key_file" { + description = "Path to service account key file (JSON)" + type = string + default = "~/.config/yandex-cloud/key.json" +} + +variable "zone" { + description = "Yandex Cloud availability zone" + type = string + default = "ru-central1-a" +} + +# VM configuration +variable "vm_name" { + description = "Name of the virtual machine" + type = string + default = "lab04-vm" +} + +# SSH configuration +variable "ssh_user" { + description = "SSH username for VM access" + type = string + default = "ubuntu" +} + +variable "ssh_public_key_path" { + description = "Path to SSH public key file" + type = string + default = "~/.ssh/id_rsa.pub" +} + +variable "ssh_allowed_cidr" { + description = "CIDR allowed to SSH, e.g. 203.0.113.10/32" + type = string +} + +# GitHub configuration (for bonus task) +variable "github_token" { + description = "GitHub personal access token" + type = string + sensitive = true + default = "" +} + +variable "github_owner" { + description = "GitHub repository owner (username or organization)" + type = string + default = "" +} \ No newline at end of file