diff --git a/.promo_web/src/generated/site-data.json b/.promo_web/src/generated/site-data.json index f56770f..6e52e18 100644 --- a/.promo_web/src/generated/site-data.json +++ b/.promo_web/src/generated/site-data.json @@ -1,5 +1,5 @@ { - "generatedAt": "2026-06-10T09:00:14.690Z", + "generatedAt": "2026-06-19T18:05:15.920Z", "requirements": { "maxLab": { "memory": 2048, @@ -14,15 +14,15 @@ }, "labs": [ { - "id": "nginx-bad-gateway", - "title": "Nginx Bad Gateway", - "summary": "Scenario The monitoring team has alerted us that the company's main landing page is down, showing a \"502 Bad Gateway\" error.", - "category": "web", - "difficulty": "beginner", - "labId": "nginx-bad-gateway", - "updatedAt": "2026-06-10T09:00:13.093Z", + "id": "time-drift-broken", + "title": "Time Drift Breaks TLS and Package Operations", + "summary": "Scenario A maintenance script stopped refreshing packages after the machine's clock drifted far into the past.", + "category": "linux", + "difficulty": "intermediate", + "labId": "time-drift-broken", + "updatedAt": "2026-06-19T18:05:15.908Z", "commit": "", - "url": "https://github.com/HimanM/BrokenOps/tree/main/labs/nginx-bad-gateway", + "url": "https://github.com/HimanM/BrokenOps/tree/main/labs/time-drift-broken", "resources": { "memory": 1024, "cpu": 1, @@ -30,15 +30,15 @@ } }, { - "id": "ca-certificates-missing", - "title": "HTTPS Downloads Fail Because CA Certificates Are Missing", - "summary": "Scenario A bootstrap service is supposed to download and install a tiny internal package from a local HTTPS endpoint.", + "id": "rsyslog-forwarding-broken", + "title": "Rsyslog Remote Forwarding Not Working", + "summary": "Scenario Local logs are written normally, but they never reach the central collector.", "category": "linux", - "difficulty": "intermediate", - "labId": "ca-certificates-missing", - "updatedAt": "2026-06-10T13:36:17+05:30", - "commit": "620c6961a1686b5117c37f67ea7fa6d31814bfac", - "url": "https://github.com/HimanM/BrokenOps/tree/main/labs/ca-certificates-missing", + "difficulty": "advanced", + "labId": "rsyslog-forwarding-broken", + "updatedAt": "2026-06-19T18:05:15.817Z", + "commit": "", + "url": "https://github.com/HimanM/BrokenOps/tree/main/labs/rsyslog-forwarding-broken", "resources": { "memory": 1024, "cpu": 1, @@ -46,95 +46,95 @@ } }, { - "id": "cron-broken", - "title": "Cron Job Never Executes", - "summary": "Scenario A system administrator configured a cron job to run a cleanup script /opt/cleanup.sh every minute.", - "category": "linux", - "difficulty": "beginner", - "labId": "cron-broken", - "updatedAt": "2026-06-10T13:36:17+05:30", - "commit": "620c6961a1686b5117c37f67ea7fa6d31814bfac", - "url": "https://github.com/HimanM/BrokenOps/tree/main/labs/cron-broken", + "id": "compose-service-dns-breaks-after-rename", + "title": "Compose Service DNS Breaks After Rename", + "summary": "Scenario A Compose-based proxy still references an old backend hostname after the service was renamed.", + "category": "docker", + "difficulty": "intermediate", + "labId": "compose-service-dns-breaks-after-rename", + "updatedAt": "2026-06-11T07:08:39Z", + "commit": "401e96d5c6abece75cc8c03ae4e3e4978b9f1ad1", + "url": "https://github.com/HimanM/BrokenOps/tree/main/labs/compose-service-dns-breaks-after-rename", "resources": { - "memory": 1024, - "cpu": 1, + "memory": 2048, + "cpu": 2, "disk": 10 } }, { - "id": "dns-broken", - "title": "DNS Resolution Failure", - "summary": "Scenario A teammate changed DNS settings on this server.", - "category": "linux", + "id": "docker-entrypoint-flags", + "title": "Container Entrypoint Uses Wrong Command Flags", + "summary": "Scenario A container image builds, but the application exits right away because the default command still uses an obsolete flag name.", + "category": "docker", "difficulty": "beginner", - "labId": "dns-broken", - "updatedAt": "2026-06-10T13:36:17+05:30", - "commit": "620c6961a1686b5117c37f67ea7fa6d31814bfac", - "url": "https://github.com/HimanM/BrokenOps/tree/main/labs/dns-broken", + "labId": "docker-entrypoint-flags", + "updatedAt": "2026-06-11T07:08:39Z", + "commit": "401e96d5c6abece75cc8c03ae4e3e4978b9f1ad1", + "url": "https://github.com/HimanM/BrokenOps/tree/main/labs/docker-entrypoint-flags", "resources": { - "memory": 1024, - "cpu": 1, + "memory": 2048, + "cpu": 2, "disk": 10 } }, { - "id": "docker-broken", - "title": "Docker Daemon Crash", - "summary": "Scenario A developer was trying to configure the Docker daemon to use a custom data root (/var/lib/docker_custom), but something went wrong and now the docker service refuses to st", - "category": "containers", - "difficulty": "advanced", - "labId": "docker-broken", - "updatedAt": "2026-06-10T13:36:17+05:30", - "commit": "620c6961a1686b5117c37f67ea7fa6d31814bfac", - "url": "https://github.com/HimanM/BrokenOps/tree/main/labs/docker-broken", + "id": "docker-loopback-bind", + "title": "Container Port Binds Only to Loopback", + "summary": "Scenario The container starts, but the Open Port path cannot reach it because the service only listens on localhost inside the container.", + "category": "docker", + "difficulty": "intermediate", + "labId": "docker-loopback-bind", + "updatedAt": "2026-06-11T07:08:39Z", + "commit": "401e96d5c6abece75cc8c03ae4e3e4978b9f1ad1", + "url": "https://github.com/HimanM/BrokenOps/tree/main/labs/docker-loopback-bind", "resources": { - "memory": 1024, - "cpu": 1, + "memory": 2048, + "cpu": 2, "disk": 10 } }, { - "id": "docker-restart-loop", - "title": "Container Restart Loop", - "summary": "Scenario A web application container web-app has been deployed on this server.", - "category": "containers", + "id": "docker-multi-stage-copy-path", + "title": "Docker Multi-Stage COPY Path", + "summary": "Scenario A container image stopped building after a refactor changed the artifact path in a multi-stage Dockerfile.", + "category": "docker", "difficulty": "intermediate", - "labId": "docker-restart-loop", - "updatedAt": "2026-06-10T13:36:17+05:30", - "commit": "620c6961a1686b5117c37f67ea7fa6d31814bfac", - "url": "https://github.com/HimanM/BrokenOps/tree/main/labs/docker-restart-loop", + "labId": "docker-multi-stage-copy-path", + "updatedAt": "2026-06-11T07:08:39Z", + "commit": "401e96d5c6abece75cc8c03ae4e3e4978b9f1ad1", + "url": "https://github.com/HimanM/BrokenOps/tree/main/labs/docker-multi-stage-copy-path", "resources": { - "memory": 1024, - "cpu": 1, + "memory": 2048, + "cpu": 2, "disk": 10 } }, { - "id": "fstab-broken", - "title": "Filesystem Missing After Reboot", - "summary": "Scenario A teammate configured a secondary storage volume (modeled as /opt/data.img) to automatically mount at /mnt/data on boot.", - "category": "linux", - "difficulty": "intermediate", - "labId": "fstab-broken", - "updatedAt": "2026-06-10T13:36:17+05:30", - "commit": "620c6961a1686b5117c37f67ea7fa6d31814bfac", - "url": "https://github.com/HimanM/BrokenOps/tree/main/labs/fstab-broken", + "id": "mysql-max-connections", + "title": "MySQL Max Connections Too Low", + "summary": "Scenario The application team reports that their service is frequently crashing with \"Too many connections\" errors from the MySQL database during peak traffic.", + "category": "database", + "difficulty": "advanced", + "labId": "mysql-max-connections", + "updatedAt": "2026-06-11T10:34:25+05:30", + "commit": "b808c27234405689f673dc646fedd176d90cdc4c", + "url": "https://github.com/HimanM/BrokenOps/tree/main/labs/mysql-max-connections", "resources": { - "memory": 1024, - "cpu": 1, - "disk": 10 + "memory": 2048, + "cpu": 2, + "disk": 20 } }, { - "id": "full-disk", - "title": "The Full Filesystem", - "summary": "Scenario A critical service on this system has failed to start or is operating extremely slowly.", - "category": "linux", - "difficulty": "beginner", - "labId": "full-disk", - "updatedAt": "2026-06-10T13:36:17+05:30", - "commit": "620c6961a1686b5117c37f67ea7fa6d31814bfac", - "url": "https://github.com/HimanM/BrokenOps/tree/main/labs/full-disk", + "id": "nginx-rate-limiting", + "title": "Nginx Aggressive Rate Limiting", + "summary": "Scenario Users are reporting that the /api/ endpoint of our web application is intermittently failing with 429 Too Many Requests errors, even when they aren't sending many requests", + "category": "web", + "difficulty": "advanced", + "labId": "nginx-rate-limiting", + "updatedAt": "2026-06-11T10:34:25+05:30", + "commit": "b808c27234405689f673dc646fedd176d90cdc4c", + "url": "https://github.com/HimanM/BrokenOps/tree/main/labs/nginx-rate-limiting", "resources": { "memory": 1024, "cpu": 1, diff --git a/labs/rsyslog-forwarding-broken/cloud-init.yaml b/labs/rsyslog-forwarding-broken/cloud-init.yaml new file mode 100644 index 0000000..8accae0 --- /dev/null +++ b/labs/rsyslog-forwarding-broken/cloud-init.yaml @@ -0,0 +1,31 @@ +#cloud-config +packages: + - rsyslog + - socat +write_files: + - path: /etc/systemd/system/collector.service + owner: root:root + permissions: '0644' + content: | + [Unit] + Description=BrokenOps syslog collector + After=network.target + + [Service] + ExecStart=/usr/bin/socat -u TCP-LISTEN:1514,reuseaddr,fork OPEN:/var/log/collector.log,append + Restart=always + RestartSec=1 + + [Install] + WantedBy=multi-user.target + - path: /etc/rsyslog.d/50-forward.conf + owner: root:root + permissions: '0644' + content: | + *.* @127.0.0.1:1514 +runcmd: + - systemctl daemon-reload + - touch /var/log/collector.log + - systemctl enable --now collector.service + - systemctl enable --now rsyslog + - systemctl restart rsyslog diff --git a/labs/rsyslog-forwarding-broken/lab.yaml b/labs/rsyslog-forwarding-broken/lab.yaml new file mode 100644 index 0000000..f624d92 --- /dev/null +++ b/labs/rsyslog-forwarding-broken/lab.yaml @@ -0,0 +1,25 @@ +id: rsyslog-forwarding-broken +name: Rsyslog Remote Forwarding Not Working +category: linux +difficulty: advanced +vm: + name: rsyslog-lab + image: ubuntu-24.04-base.qcow2 + cpu: 1 + memory: 1024 + disk: 10G +cloud_init: cloud-init.yaml +verify_script: verify.sh +description: + summary: "Local logs are written, but they never reach the remote collector." + story: "A systems administrator set up remote log forwarding so the local server can ship syslog events to a central collector. The collector is running, but forwarded messages never arrive. Fix the rsyslog forwarding rule so remote logs are delivered again." + objectives: + - Inspect the rsyslog forwarding configuration + - Find why messages are not reaching the collector + - Fix the forwarding protocol or endpoint and confirm receipt on the collector + - Ensure rsyslog remains healthy after the change +tags: + - linux + - rsyslog + - logging + - networking diff --git a/labs/rsyslog-forwarding-broken/question.md b/labs/rsyslog-forwarding-broken/question.md new file mode 100644 index 0000000..e2850f2 --- /dev/null +++ b/labs/rsyslog-forwarding-broken/question.md @@ -0,0 +1,19 @@ +### Scenario + +Local logs are written normally, but they never reach the central collector. The collector service is up, yet forwarded syslog messages are missing. + +Fix the forwarding rule so the remote collector receives the logs again. + +### Objective + +1. Inspect the rsyslog forwarding configuration. +2. Identify why messages are not reaching the collector. +3. Fix the forwarding protocol or endpoint and confirm receipt on the collector. +4. Verify rsyslog remains healthy after the change. + +### Useful Commands + +- `systemctl status rsyslog` +- `rsyslogd -N1` +- `cat /etc/rsyslog.d/50-forward.conf` +- `logger -t brokenops-rsyslog "test message"` diff --git a/labs/rsyslog-forwarding-broken/solution.md b/labs/rsyslog-forwarding-broken/solution.md new file mode 100644 index 0000000..0e84fd9 --- /dev/null +++ b/labs/rsyslog-forwarding-broken/solution.md @@ -0,0 +1,31 @@ +### The Issue + +The forwarding rule used the wrong transport. The collector listens on TCP, but the rsyslog rule was sending UDP packets, so the messages never arrived. + +### Step-by-Step Fix + +1. **Inspect the forwarding rule**: + ```bash + cat /etc/rsyslog.d/50-forward.conf + ``` + You should see a single `@` in the forwarding target. + +2. **Validate the configuration**: + ```bash + sudo rsyslogd -N1 + ``` + That confirms the syntax is valid, but it does not guarantee the collector will receive the message. + +3. **Switch the rule to TCP forwarding**: + Change the rule to use a double `@@`: + ```bash + sudo sh -c 'printf "*.* @@127.0.0.1:1514\n" > /etc/rsyslog.d/50-forward.conf' + sudo systemctl restart rsyslog + ``` + +4. **Send a test log message**: + ```bash + logger -t brokenops-rsyslog "rsyslog forwarding test" + tail -n 20 /var/log/collector.log + ``` + Once the transport matches the collector, the message shows up in the log file. diff --git a/labs/rsyslog-forwarding-broken/solution.sh b/labs/rsyslog-forwarding-broken/solution.sh new file mode 100755 index 0000000..1fe64e6 --- /dev/null +++ b/labs/rsyslog-forwarding-broken/solution.sh @@ -0,0 +1,8 @@ +#!/bin/bash +set -euo pipefail + +cat > /etc/rsyslog.d/50-forward.conf <<'EOF' +*.* @@127.0.0.1:1514 +EOF + +systemctl restart rsyslog diff --git a/labs/rsyslog-forwarding-broken/verify.sh b/labs/rsyslog-forwarding-broken/verify.sh new file mode 100644 index 0000000..77d345f --- /dev/null +++ b/labs/rsyslog-forwarding-broken/verify.sh @@ -0,0 +1,32 @@ +#!/bin/bash +set -euo pipefail + +LOG_FILE=/var/log/collector.log +TEST_TAG=brokenops-rsyslog +TEST_MESSAGE="rsyslog forwarding test $(date -Is)" + +if ! systemctl is-active --quiet rsyslog; then + echo "FAILURE: rsyslog is not running." + exit 1 +fi + +if ! rsyslogd -N1 >/tmp/rsyslog-syntax.log 2>&1; then + cat /tmp/rsyslog-syntax.log + echo "FAILURE: rsyslog configuration still has syntax errors." + exit 1 +fi + +logger -t "$TEST_TAG" "$TEST_MESSAGE" +sleep 2 + +if [ ! -f "$LOG_FILE" ]; then + echo "FAILURE: The collector log was not created." + exit 1 +fi + +if ! grep -q "$TEST_TAG" "$LOG_FILE" || ! grep -q "$TEST_MESSAGE" "$LOG_FILE"; then + echo "FAILURE: The forwarded log message never reached the collector." + exit 1 +fi + +echo "SUCCESS: rsyslog forwards messages to the remote collector successfully." diff --git a/labs/time-drift-broken/cloud-init.yaml b/labs/time-drift-broken/cloud-init.yaml new file mode 100644 index 0000000..e9fa672 --- /dev/null +++ b/labs/time-drift-broken/cloud-init.yaml @@ -0,0 +1,75 @@ +#cloud-config +packages: + - chrony + - openssl + - gzip +runcmd: + - mkdir -p /srv/time-repo/dists/stable/main/binary-amd64 + - touch /srv/time-repo/dists/stable/main/binary-amd64/Packages + - gzip -n -f /srv/time-repo/dists/stable/main/binary-amd64/Packages + - | + repo=/srv/time-repo + pkg="$repo/dists/stable/main/binary-amd64/Packages.gz" + size=$(stat -c%s "$pkg") + md5=$(md5sum "$pkg" | awk '{print $1}') + sha256=$(sha256sum "$pkg" | awk '{print $1}') + cat > "$repo/dists/stable/Release" < /etc/time-lab/certs/openssl.cnf <<'EOF' + [req] + default_bits = 2048 + prompt = no + default_md = sha256 + distinguished_name = dn + x509_extensions = v3_req + + [dn] + CN = 127.0.0.1 + + [v3_req] + subjectAltName = @alt_names + + [alt_names] + DNS.1 = localhost + IP.1 = 127.0.0.1 + EOF + - openssl req -x509 -nodes -newkey rsa:2048 -days 365 -keyout /etc/time-lab/certs/server.key -out /etc/time-lab/certs/server.crt -config /etc/time-lab/certs/openssl.cnf + - cp /etc/time-lab/certs/server.crt /usr/local/share/ca-certificates/time-repo.crt + - update-ca-certificates + - | + cat > /etc/systemd/system/time-repo.service <<'EOF' + [Unit] + Description=BrokenOps local HTTPS apt mirror + After=network.target + + [Service] + WorkingDirectory=/srv/time-repo + ExecStart=/usr/bin/openssl s_server -quiet -accept 8443 -cert /etc/time-lab/certs/server.crt -key /etc/time-lab/certs/server.key -WWW + Restart=always + + [Install] + WantedBy=multi-user.target + EOF + - systemctl daemon-reload + - systemctl enable --now time-repo.service + - | + cat > /etc/apt/sources.list.d/time-repo.list <<'EOF' + deb [trusted=yes] https://127.0.0.1:8443 stable main + EOF + - systemctl disable --now chrony || true + - timedatectl set-ntp false || true + - date -u -s "$(date -u -d '2 days ago' '+%Y-%m-%d %H:%M:%S')" diff --git a/labs/time-drift-broken/lab.yaml b/labs/time-drift-broken/lab.yaml new file mode 100644 index 0000000..5061cf9 --- /dev/null +++ b/labs/time-drift-broken/lab.yaml @@ -0,0 +1,24 @@ +id: time-drift-broken +name: Time Drift Breaks TLS and Package Operations +category: linux +difficulty: intermediate +vm: + name: time-drift-lab + image: ubuntu-24.04-base.qcow2 + cpu: 1 + memory: 1024 + disk: 10G +cloud_init: cloud-init.yaml +verify_script: verify.sh +description: + summary: "TLS requests and package updates fail because the system clock is badly skewed." + story: "A maintenance script stopped refreshing packages after the machine's clock drifted far into the past. HTTPS checks now fail with certificate time errors, and apt can no longer reach a local package mirror. Restore correct time synchronization." + objectives: + - Inspect the system clock and time synchronization status + - Find why the machine time is skewed far into the past + - Restore proper time sync and confirm TLS and apt operations work again +tags: + - linux + - time + - tls + - apt diff --git a/labs/time-drift-broken/question.md b/labs/time-drift-broken/question.md new file mode 100644 index 0000000..bd6b4f7 --- /dev/null +++ b/labs/time-drift-broken/question.md @@ -0,0 +1,18 @@ +### Scenario + +A maintenance script stopped refreshing packages after the machine's clock drifted far into the past. HTTPS requests now fail with certificate time errors, and the local package mirror no longer updates cleanly. + +Restore the system clock and time synchronization so TLS checks and package operations work again. + +### Objective + +1. Inspect the system clock and time synchronization status. +2. Find why the machine time is skewed far into the past. +3. Restore proper time sync and confirm TLS and apt operations work again. + +### Useful Commands + +- `timedatectl status` +- `systemctl status chrony` +- `openssl x509 -in /etc/time-lab/certs/server.crt -noout -startdate` +- `apt-get update -o Dir::Etc::sourcelist=/etc/apt/sources.list.d/time-repo.list -o Dir::Etc::sourceparts=-` diff --git a/labs/time-drift-broken/solution.md b/labs/time-drift-broken/solution.md new file mode 100644 index 0000000..ee8c387 --- /dev/null +++ b/labs/time-drift-broken/solution.md @@ -0,0 +1,31 @@ +### The Issue + +The machine clock was pushed far into the past and the time sync service was disabled. Because of that, TLS certificates were seen as not yet valid, and package updates against the local HTTPS mirror failed. + +### Step-by-Step Fix + +1. **Check the current clock and service state**: + ```bash + timedatectl status + systemctl status chrony + ``` + +2. **Inspect the local mirror certificate**: + ```bash + openssl x509 -in /etc/time-lab/certs/server.crt -noout -startdate + ``` + The certificate start date gives you a trustworthy reference for the expected time window. + +3. **Bring the clock back into range**: + If chrony is disabled, re-enable it and correct the clock: + ```bash + sudo systemctl enable --now chrony + sudo date -u -s "$(openssl x509 -in /etc/time-lab/certs/server.crt -noout -startdate | cut -d= -f2)" + ``` + If `chronyc makestep` is available and the system already has time sources, that is also acceptable. + +4. **Verify package access over HTTPS**: + ```bash + sudo apt-get update -o Dir::Etc::sourcelist=/etc/apt/sources.list.d/time-repo.list -o Dir::Etc::sourceparts=- + ``` + Once the clock is correct, the TLS handshake succeeds and the local apt mirror updates normally. diff --git a/labs/time-drift-broken/solution.sh b/labs/time-drift-broken/solution.sh new file mode 100755 index 0000000..d659f87 --- /dev/null +++ b/labs/time-drift-broken/solution.sh @@ -0,0 +1,6 @@ +#!/bin/bash +set -euo pipefail + +systemctl enable --now chrony >/dev/null 2>&1 || true +cert_date=$(openssl x509 -in /etc/time-lab/certs/server.crt -noout -startdate | cut -d= -f2) +date -u -s "$cert_date" >/dev/null diff --git a/labs/time-drift-broken/verify.sh b/labs/time-drift-broken/verify.sh new file mode 100644 index 0000000..8c4c283 --- /dev/null +++ b/labs/time-drift-broken/verify.sh @@ -0,0 +1,33 @@ +#!/bin/bash +set -euo pipefail + +CERT=/etc/time-lab/certs/server.crt +SOURCE_LIST=/etc/apt/sources.list.d/time-repo.list +LOG=/tmp/time-repo-update.log + +if ! systemctl is-active --quiet chrony; then + echo "FAILURE: chrony is not running." + exit 1 +fi + +if [ ! -f "$CERT" ]; then + echo "FAILURE: The local repository certificate is missing." + exit 1 +fi + +START_DATE=$(openssl x509 -in "$CERT" -noout -startdate | cut -d= -f2) +CERT_EPOCH=$(date -u -d "$START_DATE" +%s) +NOW_EPOCH=$(date -u +%s) +DIFF=$((NOW_EPOCH - CERT_EPOCH)) +if [ ${DIFF#-} -gt 300 ]; then + echo "FAILURE: The system clock is still skewed by more than 5 minutes." + exit 1 +fi + +if ! apt-get update -o Dir::Etc::sourcelist="$SOURCE_LIST" -o Dir::Etc::sourceparts=- -o APT::Get::List-Cleanup=0 >"$LOG" 2>&1; then + cat "$LOG" + echo "FAILURE: apt update against the local HTTPS repository still fails." + exit 1 +fi + +echo "SUCCESS: Time sync is restored and the local HTTPS apt repository updates successfully."