[SOU-650] 디스크 모니터링 로직 개선 및 임계값 이중화

deploy/shared/discord-notify.sh

@@ -174,6 +174,37 @@ notify_server_up() {
     curl -s -H "Content-Type: application/json" -X POST -d "$PAYLOAD" "$DISCORD_WEBHOOK_URL" > /dev/null
 }
 
+notify_disk_critical() {
+    local env=${1:-"dev"}
+    local disk_usage=$2
+    local disk_percent=$3
+    local timestamp=$(date '+%Y-%m-%d %H:%M:%S')
+    [ -z "$DISCORD_WEBHOOK_URL" ] && return
+
+    PAYLOAD=$(jq -n \
+      --arg env "$env" \
+      --arg du "$disk_usage" \
+      --arg dp "$disk_percent" \
+      --arg ts "$timestamp" \
+      --argjson color "$COLOR_RED" \
+      '{
+        username: "Souzip Bot",
+        content: "@here 디스크 공간 부족",
+        embeds: [{
+          title: ("디스크 사용량 경고 [" + $env + "]"),
+          description: "디스크 사용량이 95%를 초과했습니다.",
+          color: $color,
+          fields: [
+            {name: "환경", value: $env, inline: true},
+            {name: "시간", value: $ts, inline: true},
+            {name: "디스크 사용량", value: "\($du) (\($dp)%)", inline: false}
+          ]
+        }]
+      }')
+
+    curl -s -H "Content-Type: application/json" -X POST -d "$PAYLOAD" "$DISCORD_WEBHOOK_URL" > /dev/null
+}
+
 notify_disk_warning() {
     local env=${1:-"dev"}
     local disk_usage=$2
@@ -192,7 +223,7 @@ notify_disk_warning() {
         content: "@here 디스크 공간 부족",
         embeds: [{
           title: ("디스크 사용량 경고 [" + $env + "]"),
-          description: "디스크 사용량이 임계값을 초과했습니다.",
+          description: "디스크 사용량이 90%를 초과했습니다.",
           color: $color,
           fields: [
             {name: "환경", value: $env, inline: true},

deploy/shared/system-monitor.sh

@@ -10,7 +10,8 @@ else
 fi
 DEPLOY_DIR="$WORK_DIR/deploy/$ENV"
 STATUS_FILE="$DEPLOY_DIR/.system-status"
-DISK_THRESHOLD=80
+DISK_THRESHOLD=90
+DISK_CRITICAL_THRESHOLD=95
 MEMORY_THRESHOLD=90
 MAX_HEALTH_FAILURES=3
 
@@ -30,11 +31,13 @@ fi
 
 if [ -f "$STATUS_FILE" ]; then
     source "$STATUS_FILE"
+    DISK_CRITICAL_SENT="${DISK_CRITICAL_SENT:-false}"
 else
     HEALTH_STATUS="up"
     HEALTH_FAILURE_COUNT=0
     HEALTH_DOWN_SINCE=""
     DISK_WARNING_SENT=false
+    DISK_CRITICAL_SENT=false
     MEMORY_WARNING_SENT=false
     CONTAINER_WARNING_SENT=false
 fi
@@ -98,6 +101,16 @@ fi
 
 DISK_USAGE=$(df -h / | awk 'NR==2 {print $3 " / " $2}')
 DISK_PERCENT=$(df / | awk 'NR==2 {print int($5)}')
+if [ $DISK_PERCENT -ge $DISK_CRITICAL_THRESHOLD ]; then
+    if [ "$DISK_CRITICAL_SENT" = "false" ]; then
+        if [ ! -z "$DISCORD_WEBHOOK_URL" ]; then
+            notify_disk_critical "$ENV" "$DISK_USAGE" "$DISK_PERCENT"
+        fi
+        DISK_CRITICAL_SENT=true
+    fi
+else
+    DISK_CRITICAL_SENT=false
+fi
 if [ $DISK_PERCENT -ge $DISK_THRESHOLD ]; then
     if [ "$DISK_WARNING_SENT" = "false" ]; then
         if [ ! -z "$DISCORD_WEBHOOK_URL" ]; then
@@ -134,13 +147,17 @@ else
     CONTAINER_WARNING_SENT=false
 fi
 
-TEMP_STATUS_FILE=$(mktemp "${STATUS_FILE}.tmp.XXXXXX")
-cat > "$TEMP_STATUS_FILE" << EOF
+if TEMP_STATUS_FILE=$(mktemp "${STATUS_FILE}.tmp.XXXXXX" 2>/dev/null); then
+    cat > "$TEMP_STATUS_FILE" << EOF
 HEALTH_STATUS="$HEALTH_STATUS"
 HEALTH_FAILURE_COUNT=$HEALTH_FAILURE_COUNT
 HEALTH_DOWN_SINCE="$HEALTH_DOWN_SINCE"
 DISK_WARNING_SENT=$DISK_WARNING_SENT
+DISK_CRITICAL_SENT=$DISK_CRITICAL_SENT
 MEMORY_WARNING_SENT=$MEMORY_WARNING_SENT
 CONTAINER_WARNING_SENT=$CONTAINER_WARNING_SENT
 EOF
-mv "$TEMP_STATUS_FILE" "$STATUS_FILE"
+    mv "$TEMP_STATUS_FILE" "$STATUS_FILE"
+else
+    echo "[WARN] STATUS_FILE 업데이트 실패 (디스크 공간 부족)"
+fi