From f476b46ad2dc5bcfe4c169b364dd1efb48035890 Mon Sep 17 00:00:00 2001 From: bum0w0 Date: Fri, 12 Jun 2026 22:59:10 +0900 Subject: [PATCH] =?UTF-8?q?fix:=20=EB=94=94=EC=8A=A4=ED=81=AC=20=EB=AA=A8?= =?UTF-8?q?=EB=8B=88=ED=84=B0=EB=A7=81=20=EB=A1=9C=EC=A7=81=20=EA=B0=9C?= =?UTF-8?q?=EC=84=A0=20=EB=B0=8F=20=EC=9E=84=EA=B3=84=EA=B0=92=20=EC=9D=B4?= =?UTF-8?q?=EC=A4=91=ED=99=94=20-=2080%=20=EC=B4=88=EA=B3=BC=20=ED=9B=84?= =?UTF-8?q?=20100%=EA=B9=8C=EC=A7=80=20=EC=B6=94=EA=B0=80=20=EC=95=8C?= =?UTF-8?q?=EB=A6=BC=EC=9D=B4=20=EC=97=86=EB=8D=98=20=EB=8B=A8=EC=9D=BC=20?= =?UTF-8?q?=EC=9E=84=EA=B3=84=EA=B0=92=20=EA=B5=AC=EC=A1=B0=20=EA=B0=9C?= =?UTF-8?q?=EC=84=A0=20-=20=EA=B2=BD=EA=B3=A0=20=EC=9E=84=EA=B3=84?= =?UTF-8?q?=EA=B0=92=2090%,=20=EA=B8=B4=EA=B8=89=20=EC=9E=84=EA=B3=84?= =?UTF-8?q?=EA=B0=92=2095%=EB=A1=9C=20=EC=9D=B4=EC=A4=91=20=EB=8B=A8?= =?UTF-8?q?=EA=B3=84=20=EC=95=8C=EB=A6=BC=20=EA=B5=AC=EC=84=B1=20-=20?= =?UTF-8?q?=EB=94=94=EC=8A=A4=ED=81=AC=20100%=20=EC=8B=9C=20mktemp=20?= =?UTF-8?q?=EC=8B=A4=ED=8C=A8=EB=A1=9C=20=EC=8A=A4=ED=81=AC=EB=A6=BD?= =?UTF-8?q?=ED=8A=B8=EA=B0=80=20=EC=A2=85=EB=A3=8C=EB=90=98=EB=8D=98=20?= =?UTF-8?q?=EB=AC=B8=EC=A0=9C=20=EC=88=98=EC=A0=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- deploy/shared/discord-notify.sh | 33 ++++++++++++++++++++++++++++++++- deploy/shared/system-monitor.sh | 25 +++++++++++++++++++++---- 2 files changed, 53 insertions(+), 5 deletions(-) diff --git a/deploy/shared/discord-notify.sh b/deploy/shared/discord-notify.sh index 3ae2a0bb..00dcd5d3 100644 --- a/deploy/shared/discord-notify.sh +++ b/deploy/shared/discord-notify.sh @@ -174,6 +174,37 @@ notify_server_up() { curl -s -H "Content-Type: application/json" -X POST -d "$PAYLOAD" "$DISCORD_WEBHOOK_URL" > /dev/null } +notify_disk_critical() { + local env=${1:-"dev"} + local disk_usage=$2 + local disk_percent=$3 + local timestamp=$(date '+%Y-%m-%d %H:%M:%S') + [ -z "$DISCORD_WEBHOOK_URL" ] && return + + PAYLOAD=$(jq -n \ + --arg env "$env" \ + --arg du "$disk_usage" \ + --arg dp "$disk_percent" \ + --arg ts "$timestamp" \ + --argjson color "$COLOR_RED" \ + '{ + username: "Souzip Bot", + content: "@here 디스크 공간 부족", + embeds: [{ + title: ("디스크 사용량 경고 [" + $env + "]"), + description: "디스크 사용량이 95%를 초과했습니다.", + color: $color, + fields: [ + {name: "환경", value: $env, inline: true}, + {name: "시간", value: $ts, inline: true}, + {name: "디스크 사용량", value: "\($du) (\($dp)%)", inline: false} + ] + }] + }') + + curl -s -H "Content-Type: application/json" -X POST -d "$PAYLOAD" "$DISCORD_WEBHOOK_URL" > /dev/null +} + notify_disk_warning() { local env=${1:-"dev"} local disk_usage=$2 @@ -192,7 +223,7 @@ notify_disk_warning() { content: "@here 디스크 공간 부족", embeds: [{ title: ("디스크 사용량 경고 [" + $env + "]"), - description: "디스크 사용량이 임계값을 초과했습니다.", + description: "디스크 사용량이 90%를 초과했습니다.", color: $color, fields: [ {name: "환경", value: $env, inline: true}, diff --git a/deploy/shared/system-monitor.sh b/deploy/shared/system-monitor.sh index 014d0a1f..b9c02b1a 100644 --- a/deploy/shared/system-monitor.sh +++ b/deploy/shared/system-monitor.sh @@ -10,7 +10,8 @@ else fi DEPLOY_DIR="$WORK_DIR/deploy/$ENV" STATUS_FILE="$DEPLOY_DIR/.system-status" -DISK_THRESHOLD=80 +DISK_THRESHOLD=90 +DISK_CRITICAL_THRESHOLD=95 MEMORY_THRESHOLD=90 MAX_HEALTH_FAILURES=3 @@ -30,11 +31,13 @@ fi if [ -f "$STATUS_FILE" ]; then source "$STATUS_FILE" + DISK_CRITICAL_SENT="${DISK_CRITICAL_SENT:-false}" else HEALTH_STATUS="up" HEALTH_FAILURE_COUNT=0 HEALTH_DOWN_SINCE="" DISK_WARNING_SENT=false + DISK_CRITICAL_SENT=false MEMORY_WARNING_SENT=false CONTAINER_WARNING_SENT=false fi @@ -98,6 +101,16 @@ fi DISK_USAGE=$(df -h / | awk 'NR==2 {print $3 " / " $2}') DISK_PERCENT=$(df / | awk 'NR==2 {print int($5)}') +if [ $DISK_PERCENT -ge $DISK_CRITICAL_THRESHOLD ]; then + if [ "$DISK_CRITICAL_SENT" = "false" ]; then + if [ ! -z "$DISCORD_WEBHOOK_URL" ]; then + notify_disk_critical "$ENV" "$DISK_USAGE" "$DISK_PERCENT" + fi + DISK_CRITICAL_SENT=true + fi +else + DISK_CRITICAL_SENT=false +fi if [ $DISK_PERCENT -ge $DISK_THRESHOLD ]; then if [ "$DISK_WARNING_SENT" = "false" ]; then if [ ! -z "$DISCORD_WEBHOOK_URL" ]; then @@ -134,13 +147,17 @@ else CONTAINER_WARNING_SENT=false fi -TEMP_STATUS_FILE=$(mktemp "${STATUS_FILE}.tmp.XXXXXX") -cat > "$TEMP_STATUS_FILE" << EOF +if TEMP_STATUS_FILE=$(mktemp "${STATUS_FILE}.tmp.XXXXXX" 2>/dev/null); then + cat > "$TEMP_STATUS_FILE" << EOF HEALTH_STATUS="$HEALTH_STATUS" HEALTH_FAILURE_COUNT=$HEALTH_FAILURE_COUNT HEALTH_DOWN_SINCE="$HEALTH_DOWN_SINCE" DISK_WARNING_SENT=$DISK_WARNING_SENT +DISK_CRITICAL_SENT=$DISK_CRITICAL_SENT MEMORY_WARNING_SENT=$MEMORY_WARNING_SENT CONTAINER_WARNING_SENT=$CONTAINER_WARNING_SENT EOF -mv "$TEMP_STATUS_FILE" "$STATUS_FILE" + mv "$TEMP_STATUS_FILE" "$STATUS_FILE" +else + echo "[WARN] STATUS_FILE 업데이트 실패 (디스크 공간 부족)" +fi