Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
103 changes: 103 additions & 0 deletions alerts/install_alerts.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,103 @@
#!/usr/bin/env bash
# Install droplet-side security alerting to Discord. Run as root ON the droplet.
#
# sudo ./alerts/install_alerts.sh
#
# Prompts for the webhook rather than taking it as an argument, because an
# argument lands in shell history and in /proc/<pid>/cmdline where any local
# user can read it while the command runs.
#
# The webhook is written to /etc/minecraft-alerts.env as root:root 0600, which
# is outside the git repo and outside the container. It is never committed.
set -euo pipefail

ENV_FILE="/etc/minecraft-alerts.env"
BIN="/usr/local/bin/mc-security-alert"
SRC="$(cd "$(dirname "$0")" && pwd)/security_alert.sh"

[ "$(id -u)" -eq 0 ] || { echo "run as root" >&2; exit 1; }
[ -r "$SRC" ] || { echo "cannot read $SRC" >&2; exit 1; }

echo "=== EduCraft droplet security alerts ==="

if [ -r "$ENV_FILE" ] && grep -q '^DISCORD_WEBHOOK=' "$ENV_FILE"; then
echo " webhook already configured in $ENV_FILE (leaving it alone)"
else
echo
echo " Paste the Discord webhook URL. It will not be echoed."
echo " Discord: Server Settings > Integrations > Webhooks > Copy URL"
printf ' webhook: '
read -r -s WEBHOOK
echo
case "$WEBHOOK" in
https://discord.com/api/webhooks/*|https://discordapp.com/api/webhooks/*) ;;
*) echo " that does not look like a Discord webhook URL" >&2; exit 1 ;;
esac
umask 077
printf 'DISCORD_WEBHOOK=%s\n' "$WEBHOOK" > "$ENV_FILE"
unset WEBHOOK
echo " wrote $ENV_FILE"
fi
chown root:root "$ENV_FILE"
chmod 600 "$ENV_FILE"
echo " permissions: $(stat -c '%U:%G %a' "$ENV_FILE")"

install -m 0755 -o root -g root "$SRC" "$BIN"
echo " installed $BIN"

cat > /etc/systemd/system/mc-security-alert.service <<'UNIT'
[Unit]
Description=Report droplet security events to Discord
After=network-online.target
Wants=network-online.target

[Service]
Type=oneshot
ExecStart=/usr/local/bin/mc-security-alert
# The unit reads journald, auditd and docker, so it needs root. It is hardened
# in every direction that does not break those.
NoNewPrivileges=true
PrivateTmp=true
ProtectHome=true
ProtectKernelTunables=true
ProtectControlGroups=true
RestrictSUIDSGID=true
LockPersonality=true
# The webhook must never reach the journal. Output is intentionally minimal and
# the script never prints it, but this is the belt to that braces.
StandardOutput=journal
StandardError=journal
UNIT

cat > /etc/systemd/system/mc-security-alert.timer <<'UNIT'
[Unit]
Description=Check for droplet security events every 15 minutes

[Timer]
OnBootSec=5min
OnUnitActiveSec=15min
# Spread the load so every timer on the box does not fire on the same second.
RandomizedDelaySec=60
Persistent=true

[Install]
WantedBy=timers.target
UNIT

systemctl daemon-reload
systemctl enable --now mc-security-alert.timer
echo " timer enabled:"
systemctl list-timers mc-security-alert.timer --no-pager 2>/dev/null | head -3

echo
echo " running once now to prove the wiring..."
if "$BIN"; then
echo " OK"
else
echo " the run reported a problem; check: journalctl -u mc-security-alert -n 20"
fi

echo
echo " Rotate the webhook any time by deleting it in Discord and re-running this."
echo " Check state: systemctl status mc-security-alert.timer"
echo " Recent runs: journalctl -u mc-security-alert -n 40"
202 changes: 202 additions & 0 deletions alerts/security_alert.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,202 @@
#!/usr/bin/env bash
# Post NEW security events from this droplet to Discord.
#
# Installed at /usr/local/bin/mc-security-alert, run by a systemd timer every
# 15 minutes. See alerts/install_alerts.sh.
#
# DESIGN GOAL: ZERO FALSE POSITIVES.
#
# An alert channel is only useful while a message in it means "look at this".
# The first version of this script failed that badly and is worth recording so
# the mistakes are not repeated:
# - it alerted on EVERY sudo command, so ordinary admin work buried the signal
# - it alerted on docker RestartCount > 0, which is CUMULATIVE, so after any
# deploy it fired on every single run forever until the container was
# recreated. A permanent alert is the same as no alert.
# - it alerted on whitelist edits, which are routine and expected
#
# What is watched now is deliberately narrow: things that are rare, and that a
# person would want to know about within minutes. Anything that happens during
# normal operation is NOT here, on purpose.
#
# THREAT MODEL for the credential this holds.
#
# The webhook lives in /etc/minecraft-alerts.env, root:root 0600, outside the
# git repo and outside the container. The game runs as uid 1000 with a
# read-only rootfs and all capabilities dropped, so compromising Minecraft does
# not reach it. Stealing it lets an attacker post to a private Discord channel
# and spam it: annoying, not dangerous. It grants no route back into the
# droplet, because a webhook is write-only and one-directional. That asymmetry
# is why storing it here beats letting anything reach IN to collect events.
#
# Rotate by deleting the webhook in Discord and re-running install_alerts.sh.
set -uo pipefail

ENV_FILE="/etc/minecraft-alerts.env"
STATE_DIR="/var/lib/mc-alerts"
STATE="$STATE_DIR/last-run"
CONTAINER_STATE="$STATE_DIR/last-container-start"
HOST_SHORT="$(hostname -s 2>/dev/null || echo droplet)"

# Accounts expected to log in over SSH. A login by ANY other name is reported
# as high severity, because on this box that is not routine.
KNOWN_USERS="${KNOWN_SSH_USERS:-edueq9r3eiky wali root}"

[ -r "$ENV_FILE" ] || { echo "no $ENV_FILE; run install_alerts.sh" >&2; exit 1; }
# shellcheck disable=SC1090
. "$ENV_FILE"
[ -n "${DISCORD_WEBHOOK:-}" ] || { echo "DISCORD_WEBHOOK unset in $ENV_FILE" >&2; exit 1; }

mkdir -p "$STATE_DIR"; chmod 700 "$STATE_DIR"
SINCE="$(cat "$STATE" 2>/dev/null || echo "30 minutes ago")"
NOW="$(date '+%Y-%m-%d %H:%M:%S')"

# Each collector emits "severity|title|detail" lines. Empty output is the
# normal, healthy case and must stay silent.

# --- SSH logins: every one, by explicit choice. Only two people should ever
# --- log in here, so each login is genuinely worth seeing.
collect_ssh_success() {
journalctl -u ssh -u sshd --since "$SINCE" --no-pager 2>/dev/null \
| grep -E "Accepted (password|publickey|keyboard-interactive)" \
| while read -r line; do
u=$(echo "$line" | sed -nE 's/.*Accepted [a-z-]+ for ([^ ]+) from .*/\1/p')
ip=$(echo "$line" | sed -nE 's/.*from ([0-9a-fA-F.:]+) port.*/\1/p')
m=$(echo "$line" | sed -nE 's/.*Accepted ([a-z-]+) for .*/\1/p')
[ -z "$u" ] && continue
# An unrecognised username is a different class of event entirely.
if echo " $KNOWN_USERS " | grep -q " $u "; then
echo "medium|SSH login|${u} from ${ip} via ${m}"
else
echo "high|SSH login by UNKNOWN user|${u} from ${ip} via ${m}"
fi
done | sort -u | head -10
}

# --- Failed auth: port 22 is closed to everything but DigitalOcean console
# --- ranges, so genuine failures should be ~0. A burst means the firewall
# --- changed or something is inside the allowed range.
collect_ssh_failures() {
local n
n=$(journalctl -u ssh -u sshd --since "$SINCE" --no-pager 2>/dev/null \
| grep -cE "Failed password|Invalid user|Connection closed by authenticating")
[ "${n:-0}" -ge 10 ] && echo "high|SSH failure burst|${n} failed attempts since ${SINCE}"
return 0
}

# --- sudo: SENSITIVE commands only. Routine docker/journal/diagnostic work is
# --- deliberately ignored; it is the single biggest source of noise and it
# --- tells you nothing you did not already know.
collect_sudo_sensitive() {
journalctl --since "$SINCE" --no-pager 2>/dev/null \
| grep -E "sudo:.*COMMAND=" \
| grep -EI "useradd|usermod|userdel|groupadd|passwd|visudo|/etc/sudoers|authorized_keys|sshd_config|iptables|ufw|systemctl (enable|disable|mask)|chattr|/etc/shadow|/etc/passwd" \
| sed -E 's/.*sudo: *([^ ]+).*COMMAND=(.*)/high|Sensitive sudo|\1 ran \2/' \
| cut -c1-300 | sort -u | head -6
return 0
}

collect_fail2ban() {
journalctl -u fail2ban --since "$SINCE" --no-pager 2>/dev/null \
| grep -E "\bBan\b" \
| sed -E 's/.*\[([a-z-]+)\] Ban ([0-9a-fA-F.:]+).*/medium|fail2ban ban|\2 banned by \1/' \
| sort -u | head -10
return 0
}

# --- Container: compare the START TIME against the last observed one.
# --- RestartCount is cumulative and never resets, so alerting on it produced a
# --- permanent alert after the first restart. Comparing StartedAt reports each
# --- restart exactly once, which is what was actually wanted.
collect_container() {
local st started last
st=$(docker inspect minecraft-java --format '{{.State.Status}}' 2>/dev/null || echo missing)
started=$(docker inspect minecraft-java --format '{{.State.StartedAt}}' 2>/dev/null || echo unknown)
last=$(cat "$CONTAINER_STATE" 2>/dev/null || echo "")

[ "$st" != "running" ] && echo "high|Minecraft container is ${st}|expected running"
if [ -n "$last" ] && [ "$started" != "$last" ] && [ "$started" != "unknown" ]; then
echo "medium|Minecraft restarted|now up since ${started}"
fi
[ "$started" != "unknown" ] && echo "$started" > "$CONTAINER_STATE"
return 0
}

# --- Files that matter. Whitelist edits are deliberately NOT here: adding a
# --- player is routine. ops.json IS here, because granting operator is
# --- privilege escalation on the game server.
collect_sensitive_files() {
local f
for f in /etc/sudoers /etc/passwd /etc/shadow /root/.ssh/authorized_keys \
/etc/ssh/sshd_config /var/lib/docker/volumes; do
:
done
# ops.json lives in the bind-mounted data dir
if [ -f /root/minecraft/data/ops.json ] || [ -f /opt/minecraft/data/ops.json ]; then
local ops
ops=$(find /root/minecraft/data/ops.json /opt/minecraft/data/ops.json -maxdepth 0 2>/dev/null | head -1)
if [ -n "$ops" ] && [ -n "$(find "$ops" -newermt "$SINCE" 2>/dev/null)" ]; then
echo "high|Operator list changed|$(basename "$ops") modified; verify who was granted op"
fi
fi
for f in /etc/sudoers /etc/ssh/sshd_config /root/.ssh/authorized_keys; do
[ -e "$f" ] && [ -n "$(find "$f" -newermt "$SINCE" 2>/dev/null)" ] \
&& echo "high|Sensitive file changed|${f}"
done
return 0
}

collect_new_users() {
journalctl --since "$SINCE" --no-pager 2>/dev/null \
| grep -E "new user: name=|useradd\[[0-9]+\]: new user" \
| sed -E 's/.*new user: name=([^,]+).*/high|New user account|\1 created/' \
| sort -u | head -5
return 0
}

collect_disk() {
local pct
pct=$(df --output=pcent / 2>/dev/null | tail -1 | tr -dc '0-9')
[ -n "$pct" ] && [ "$pct" -ge 90 ] && echo "high|Disk nearly full|root filesystem at ${pct}%"
return 0
}

EVENTS="$( { collect_ssh_success; collect_ssh_failures; collect_sudo_sensitive;
collect_fail2ban; collect_container; collect_sensitive_files;
collect_new_users; collect_disk; } 2>/dev/null \
| grep -v '^[[:space:]]*$' )"

# Advance the checkpoint whether or not anything was found, and even if the post
# fails. Repeating an alert forever is worse than missing one: it destroys trust
# in every other alert in the channel.
echo "$NOW" > "$STATE"

[ -z "$EVENTS" ] && { echo "no security events since $SINCE"; exit 0; }

HIGH=$(echo "$EVENTS" | grep -c '^high|' || true)
COLOUR=$([ "${HIGH:-0}" -gt 0 ] && echo 11027259 || echo 11106094) # red : amber

FIELDS=$(echo "$EVENTS" | head -20 | awk -F'|' '
{ gsub(/"/,"\\\"",$2); gsub(/"/,"\\\"",$3); gsub(/\\/,"\\\\",$3);
printf "%s{\"name\":\"%s\",\"value\":\"%s\",\"inline\":false}", (NR>1?",":""), $2, substr($3,1,900) }')

COUNT=$(echo "$EVENTS" | wc -l)
PAYLOAD=$(cat <<JSON
{"embeds":[{
"title":"Security events on ${HOST_SHORT}",
"description":"${COUNT} event(s) since ${SINCE}",
"color":${COLOUR},
"fields":[${FIELDS}],
"footer":{"text":"EduCraft droplet security monitor"}
}]}
JSON
)

# Never add -v to curl: it prints the full URL, which is the credential.
if printf '%s' "$PAYLOAD" | curl -sS --fail --max-time 25 \
-H "Content-Type: application/json" -X POST -d @- "$DISCORD_WEBHOOK" >/dev/null 2>&1; then
echo "posted ${COUNT} event(s)"
else
echo "failed to post ${COUNT} event(s) to Discord" >&2
exit 1
fi
Loading