diff --git a/deploy/shared/discord-notify.sh b/deploy/shared/discord-notify.sh index 3ae2a0bb..00dcd5d3 100644 --- a/deploy/shared/discord-notify.sh +++ b/deploy/shared/discord-notify.sh @@ -174,6 +174,37 @@ notify_server_up() { curl -s -H "Content-Type: application/json" -X POST -d "$PAYLOAD" "$DISCORD_WEBHOOK_URL" > /dev/null } +notify_disk_critical() { + local env=${1:-"dev"} + local disk_usage=$2 + local disk_percent=$3 + local timestamp=$(date '+%Y-%m-%d %H:%M:%S') + [ -z "$DISCORD_WEBHOOK_URL" ] && return + + PAYLOAD=$(jq -n \ + --arg env "$env" \ + --arg du "$disk_usage" \ + --arg dp "$disk_percent" \ + --arg ts "$timestamp" \ + --argjson color "$COLOR_RED" \ + '{ + username: "Souzip Bot", + content: "@here 디스크 공간 부족", + embeds: [{ + title: ("디스크 사용량 경고 [" + $env + "]"), + description: "디스크 사용량이 95%를 초과했습니다.", + color: $color, + fields: [ + {name: "환경", value: $env, inline: true}, + {name: "시간", value: $ts, inline: true}, + {name: "디스크 사용량", value: "\($du) (\($dp)%)", inline: false} + ] + }] + }') + + curl -s -H "Content-Type: application/json" -X POST -d "$PAYLOAD" "$DISCORD_WEBHOOK_URL" > /dev/null +} + notify_disk_warning() { local env=${1:-"dev"} local disk_usage=$2 @@ -192,7 +223,7 @@ notify_disk_warning() { content: "@here 디스크 공간 부족", embeds: [{ title: ("디스크 사용량 경고 [" + $env + "]"), - description: "디스크 사용량이 임계값을 초과했습니다.", + description: "디스크 사용량이 90%를 초과했습니다.", color: $color, fields: [ {name: "환경", value: $env, inline: true}, diff --git a/deploy/shared/system-monitor.sh b/deploy/shared/system-monitor.sh index 014d0a1f..b9c02b1a 100644 --- a/deploy/shared/system-monitor.sh +++ b/deploy/shared/system-monitor.sh @@ -10,7 +10,8 @@ else fi DEPLOY_DIR="$WORK_DIR/deploy/$ENV" STATUS_FILE="$DEPLOY_DIR/.system-status" -DISK_THRESHOLD=80 +DISK_THRESHOLD=90 +DISK_CRITICAL_THRESHOLD=95 MEMORY_THRESHOLD=90 MAX_HEALTH_FAILURES=3 @@ -30,11 +31,13 @@ fi if [ -f "$STATUS_FILE" ]; then source "$STATUS_FILE" + DISK_CRITICAL_SENT="${DISK_CRITICAL_SENT:-false}" else HEALTH_STATUS="up" HEALTH_FAILURE_COUNT=0 HEALTH_DOWN_SINCE="" DISK_WARNING_SENT=false + DISK_CRITICAL_SENT=false MEMORY_WARNING_SENT=false CONTAINER_WARNING_SENT=false fi @@ -98,6 +101,16 @@ fi DISK_USAGE=$(df -h / | awk 'NR==2 {print $3 " / " $2}') DISK_PERCENT=$(df / | awk 'NR==2 {print int($5)}') +if [ $DISK_PERCENT -ge $DISK_CRITICAL_THRESHOLD ]; then + if [ "$DISK_CRITICAL_SENT" = "false" ]; then + if [ ! -z "$DISCORD_WEBHOOK_URL" ]; then + notify_disk_critical "$ENV" "$DISK_USAGE" "$DISK_PERCENT" + fi + DISK_CRITICAL_SENT=true + fi +else + DISK_CRITICAL_SENT=false +fi if [ $DISK_PERCENT -ge $DISK_THRESHOLD ]; then if [ "$DISK_WARNING_SENT" = "false" ]; then if [ ! -z "$DISCORD_WEBHOOK_URL" ]; then @@ -134,13 +147,17 @@ else CONTAINER_WARNING_SENT=false fi -TEMP_STATUS_FILE=$(mktemp "${STATUS_FILE}.tmp.XXXXXX") -cat > "$TEMP_STATUS_FILE" << EOF +if TEMP_STATUS_FILE=$(mktemp "${STATUS_FILE}.tmp.XXXXXX" 2>/dev/null); then + cat > "$TEMP_STATUS_FILE" << EOF HEALTH_STATUS="$HEALTH_STATUS" HEALTH_FAILURE_COUNT=$HEALTH_FAILURE_COUNT HEALTH_DOWN_SINCE="$HEALTH_DOWN_SINCE" DISK_WARNING_SENT=$DISK_WARNING_SENT +DISK_CRITICAL_SENT=$DISK_CRITICAL_SENT MEMORY_WARNING_SENT=$MEMORY_WARNING_SENT CONTAINER_WARNING_SENT=$CONTAINER_WARNING_SENT EOF -mv "$TEMP_STATUS_FILE" "$STATUS_FILE" + mv "$TEMP_STATUS_FILE" "$STATUS_FILE" +else + echo "[WARN] STATUS_FILE 업데이트 실패 (디스크 공간 부족)" +fi