Send droplet security events to Discord

alerts/install_alerts.sh

@@ -0,0 +1,103 @@
+#!/usr/bin/env bash
+# Install droplet-side security alerting to Discord. Run as root ON the droplet.
+#
+#   sudo ./alerts/install_alerts.sh
+#
+# Prompts for the webhook rather than taking it as an argument, because an
+# argument lands in shell history and in /proc/<pid>/cmdline where any local
+# user can read it while the command runs.
+#
+# The webhook is written to /etc/minecraft-alerts.env as root:root 0600, which
+# is outside the git repo and outside the container. It is never committed.
+set -euo pipefail
+
+ENV_FILE="/etc/minecraft-alerts.env"
+BIN="/usr/local/bin/mc-security-alert"
+SRC="$(cd "$(dirname "$0")" && pwd)/security_alert.sh"
+
+[ "$(id -u)" -eq 0 ] || { echo "run as root" >&2; exit 1; }
+[ -r "$SRC" ] || { echo "cannot read $SRC" >&2; exit 1; }
+
+echo "=== EduCraft droplet security alerts ==="
+
+if [ -r "$ENV_FILE" ] && grep -q '^DISCORD_WEBHOOK=' "$ENV_FILE"; then
+    echo "  webhook already configured in $ENV_FILE (leaving it alone)"
+else
+    echo
+    echo "  Paste the Discord webhook URL. It will not be echoed."
+    echo "  Discord: Server Settings > Integrations > Webhooks > Copy URL"
+    printf '  webhook: '
+    read -r -s WEBHOOK
+    echo
+    case "$WEBHOOK" in
+        https://discord.com/api/webhooks/*|https://discordapp.com/api/webhooks/*) ;;
+        *) echo "  that does not look like a Discord webhook URL" >&2; exit 1 ;;
+    esac
+    umask 077
+    printf 'DISCORD_WEBHOOK=%s\n' "$WEBHOOK" > "$ENV_FILE"
+    unset WEBHOOK
+    echo "  wrote $ENV_FILE"
+fi
+chown root:root "$ENV_FILE"
+chmod 600 "$ENV_FILE"
+echo "  permissions: $(stat -c '%U:%G %a' "$ENV_FILE")"
+
+install -m 0755 -o root -g root "$SRC" "$BIN"
+echo "  installed $BIN"
+
+cat > /etc/systemd/system/mc-security-alert.service <<'UNIT'
+[Unit]
+Description=Report droplet security events to Discord
+After=network-online.target
+Wants=network-online.target
+
+[Service]
+Type=oneshot
+ExecStart=/usr/local/bin/mc-security-alert
+# The unit reads journald, auditd and docker, so it needs root. It is hardened
+# in every direction that does not break those.
+NoNewPrivileges=true
+PrivateTmp=true
+ProtectHome=true
+ProtectKernelTunables=true
+ProtectControlGroups=true
+RestrictSUIDSGID=true
+LockPersonality=true
+# The webhook must never reach the journal. Output is intentionally minimal and
+# the script never prints it, but this is the belt to that braces.
+StandardOutput=journal
+StandardError=journal
+UNIT
+
+cat > /etc/systemd/system/mc-security-alert.timer <<'UNIT'
+[Unit]
+Description=Check for droplet security events every 15 minutes
+
+[Timer]
+OnBootSec=5min
+OnUnitActiveSec=15min
+# Spread the load so every timer on the box does not fire on the same second.
+RandomizedDelaySec=60
+Persistent=true
+
+[Install]
+WantedBy=timers.target
+UNIT
+
+systemctl daemon-reload
+systemctl enable --now mc-security-alert.timer
+echo "  timer enabled:"
+systemctl list-timers mc-security-alert.timer --no-pager 2>/dev/null | head -3
+
+echo
+echo "  running once now to prove the wiring..."
+if "$BIN"; then
+    echo "  OK"
+else
+    echo "  the run reported a problem; check: journalctl -u mc-security-alert -n 20"
+fi
+
+echo
+echo "  Rotate the webhook any time by deleting it in Discord and re-running this."
+echo "  Check state:   systemctl status mc-security-alert.timer"
+echo "  Recent runs:   journalctl -u mc-security-alert -n 40"

alerts/security_alert.sh

@@ -0,0 +1,146 @@
+#!/usr/bin/env bash
+# Post NEW security events from this droplet to Discord.
+#
+# Installed at /usr/local/bin/mc-security-alert and run by a systemd timer.
+# See alerts/install_alerts.sh.
+#
+# THREAT MODEL, because this puts a credential on the server.
+#
+# The webhook is a bearer token: whoever holds it can post to the channel. It
+# lives in /etc/minecraft-alerts.env, root-owned and chmod 600, OUTSIDE the git
+# repo and OUTSIDE the container. The Minecraft server runs as uid 1000 with a
+# read-only rootfs and every capability dropped, so a compromise of the game
+# does not reach it.
+#
+# What an attacker gains by stealing it: the ability to post messages to a
+# private Discord channel, and to spam it. That is annoying, not dangerous. What
+# they do NOT gain is any access back into the droplet, because a webhook is
+# write-only and one-directional. Discord cannot send commands through it.
+#
+# That asymmetry is the whole reason this is acceptable: alerts push OUT. The
+# alternative, letting something outside reach IN to collect them, would mean
+# opening a port or storing an SSH credential in CI, and both are worse.
+#
+# If the webhook is ever suspected leaked: delete it in Discord (Server
+# Settings, Integrations, Webhooks) and run install_alerts.sh again. Rotation is
+# one command and invalidates the old URL instantly.
+set -uo pipefail
+
+ENV_FILE="/etc/minecraft-alerts.env"
+STATE_DIR="/var/lib/mc-alerts"
+STATE="$STATE_DIR/last-run"
+HOSTNAME_SHORT="$(hostname -s 2>/dev/null || echo droplet)"
+
+[ -r "$ENV_FILE" ] || { echo "no $ENV_FILE; run install_alerts.sh" >&2; exit 1; }
+# shellcheck disable=SC1090
+. "$ENV_FILE"
+[ -n "${DISCORD_WEBHOOK:-}" ] || { echo "DISCORD_WEBHOOK unset in $ENV_FILE" >&2; exit 1; }
+
+mkdir -p "$STATE_DIR"; chmod 700 "$STATE_DIR"
+SINCE="$(cat "$STATE" 2>/dev/null || echo "30 minutes ago")"
+NOW="$(date '+%Y-%m-%d %H:%M:%S')"
+
+# ---------------------------------------------------------------- collectors
+# Each returns lines of "severity|title|detail". Empty output means nothing to
+# report, which is the normal case and must stay silent.
+
+collect_ssh_success() {
+    journalctl -u ssh -u sshd --since "$SINCE" --no-pager 2>/dev/null \
+      | grep -E "Accepted (password|publickey|keyboard-interactive)" \
+      | sed -E 's/.*Accepted ([a-z-]+) for ([^ ]+) from ([0-9a-f.:]+).*/high|SSH login|user \2 from \3 via \1/' \
+      | sort -u
+}
+
+collect_ssh_failures() {
+    local n
+    n=$(journalctl -u ssh -u sshd --since "$SINCE" --no-pager 2>/dev/null \
+        | grep -cE "Failed password|Invalid user|Connection closed by authenticating")
+    # Port 22 is closed to the world, so failures should be near zero. A burst
+    # means either the firewall changed or something is inside the allowed range.
+    [ "${n:-0}" -ge 10 ] && echo "medium|SSH failures|${n} failed attempts since ${SINCE}"
+}
+
+collect_sudo() {
+    journalctl --since "$SINCE" --no-pager 2>/dev/null \
+      | grep -E "sudo:.*COMMAND=" \
+      | sed -E 's/.*sudo: *([^ ]+).*COMMAND=(.*)/medium|sudo|\1 ran \2/' \
+      | cut -c1-300 | sort -u | head -10
+}
+
+collect_fail2ban() {
+    journalctl -u fail2ban --since "$SINCE" --no-pager 2>/dev/null \
+      | grep -E "\bBan\b" \
+      | sed -E 's/.*\[([a-z-]+)\] Ban ([0-9a-f.:]+).*/medium|fail2ban ban|\2 banned by \1/' \
+      | sort -u | head -10
+}
+
+collect_container() {
+    local st restarts
+    st=$(docker inspect minecraft-java --format '{{.State.Status}}' 2>/dev/null || echo missing)
+    restarts=$(docker inspect minecraft-java --format '{{.RestartCount}}' 2>/dev/null || echo 0)
+    [ "$st" != "running" ] && echo "high|Container not running|status=${st}"
+    [ "${restarts:-0}" -gt 0 ] && echo "medium|Container restarts|RestartCount=${restarts}"
+    return 0
+}
+
+collect_admin_files() {
+    # auditd watches these (see server-audit.sh). A change to the whitelist or
+    # ops file that nobody made deliberately is worth knowing about immediately.
+    command -v ausearch >/dev/null 2>&1 || return 0
+    ausearch --input-logs -ts recent -k mc-admin 2>/dev/null \
+      | grep -oE 'name="[^"]+"' | sort -u | head -5 \
+      | sed -E 's/name="(.*)"/medium|Admin file touched|\1/'
+    return 0
+}
+
+collect_disk() {
+    local pct
+    pct=$(df --output=pcent / 2>/dev/null | tail -1 | tr -dc '0-9')
+    [ -n "$pct" ] && [ "$pct" -ge 85 ] && echo "medium|Disk filling|root filesystem at ${pct}%"
+    return 0
+}
+
+# ---------------------------------------------------------------- gather
+EVENTS="$( { collect_ssh_success; collect_ssh_failures; collect_sudo;
+             collect_fail2ban; collect_container; collect_admin_files;
+             collect_disk; } 2>/dev/null | grep -v '^[[:space:]]*$' )"
+
+# Record the checkpoint whether or not anything was found, so the next run does
+# not re-report the same window. Written even on a failed post: repeating an
+# alert forever is worse than missing one, because it destroys trust in all of
+# them.
+echo "$NOW" > "$STATE"
+
+[ -z "$EVENTS" ] && { echo "no security events since $SINCE"; exit 0; }
+
+# ---------------------------------------------------------------- report
+HIGH=$(echo "$EVENTS" | grep -c '^high|' || true)
+COLOUR=$([ "${HIGH:-0}" -gt 0 ] && echo 11027259 || echo 11106094)   # red : amber
+
+FIELDS=$(echo "$EVENTS" | head -20 | awk -F'|' '
+  { gsub(/"/,"\\\"",$2); gsub(/"/,"\\\"",$3);
+    printf "%s{\"name\":\"%s\",\"value\":\"%s\",\"inline\":false}", (NR>1?",":""), $2, substr($3,1,900) }')
+
+COUNT=$(echo "$EVENTS" | wc -l)
+PAYLOAD=$(cat <<JSON
+{"embeds":[{
+  "title":"Security events on ${HOSTNAME_SHORT}",
+  "description":"${COUNT} event(s) since ${SINCE}",
+  "color":${COLOUR},
+  "fields":[${FIELDS}],
+  "footer":{"text":"EduCraft droplet security monitor"}
+}]}
+JSON
+)
+
+# --fail so a rejected post is a non-zero exit; -sS to stay quiet but still show
+# real errors. The URL is passed on stdin-free argv only here, and curl does not
+# echo it. Never add -v: it prints the full URL.
+if printf '%s' "$PAYLOAD" | curl -sS --fail --max-time 25 \
+       -H "Content-Type: application/json" -X POST -d @- "$DISCORD_WEBHOOK" >/dev/null 2>&1; then
+    echo "posted ${COUNT} event(s)"
+else
+    # Deliberately does not print the webhook or the payload.
+    echo "failed to post ${COUNT} event(s) to Discord" >&2
+    exit 1
+fi