Реализация на отдельных двух машинах с протестированным взаимодействием по проверке сообщений
This commit is contained in:
mi
@@ -0,0 +1,345 @@
|
||||
#!/usr/bin/env bash
|
||||
# HAN Chat VM metrics snapshot
|
||||
# Host: Ubuntu 24.04 + Docker + UFW + fail2ban
|
||||
#
|
||||
# Usage:
|
||||
# sudo ./han-vm-metrics.sh # human report to stdout
|
||||
# sudo ./han-vm-metrics.sh --json # JSON to stdout
|
||||
# sudo ./han-vm-metrics.sh --alert # exit 1 if thresholds breached
|
||||
# sudo ./han-vm-metrics.sh --log /var/log/han-vm-metrics.log
|
||||
#
|
||||
# Cron example (every 5 min):
|
||||
# */5 * * * * root /opt/han-chat/ops/han-vm-metrics.sh --alert --log /var/log/han-vm-metrics.log
|
||||
#
|
||||
# Disk notes:
|
||||
# Checks both / and /var/lib/docker. If Docker Root Dir is on a separate
|
||||
# mount, its fill level is reported and alerted independently.
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
MODE="text"
|
||||
LOG_FILE=""
|
||||
ALERT=0
|
||||
DISK_WARN_PCT="${DISK_WARN_PCT:-80}"
|
||||
DISK_CRIT_PCT="${DISK_CRIT_PCT:-90}"
|
||||
LOAD_WARN_MULT="${LOAD_WARN_MULT:-1.5}" # vs CPU count
|
||||
MEM_WARN_PCT="${MEM_WARN_PCT:-90}"
|
||||
CERT_WARN_DAYS="${CERT_WARN_DAYS:-21}"
|
||||
SSH_FAIL_WARN="${SSH_FAIL_WARN:-50}" # failed attempts last hour
|
||||
DOMAIN="${METRICS_DOMAIN:-chat.han0107.ru}"
|
||||
DOCKER_ROOT="${DOCKER_ROOT:-/var/lib/docker}"
|
||||
|
||||
require_value() {
|
||||
local opt="$1" val="${2:-}"
|
||||
if [[ -z "$val" || "$val" == -* ]]; then
|
||||
echo "Error: $opt requires a value" >&2
|
||||
exit 2
|
||||
fi
|
||||
}
|
||||
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case "$1" in
|
||||
--json) MODE="json"; shift ;;
|
||||
--alert) ALERT=1; shift ;;
|
||||
--log)
|
||||
require_value "$1" "${2:-}"
|
||||
LOG_FILE="$2"
|
||||
shift 2
|
||||
;;
|
||||
--domain)
|
||||
require_value "$1" "${2:-}"
|
||||
DOMAIN="$2"
|
||||
shift 2
|
||||
;;
|
||||
-h|--help)
|
||||
sed -n '2,18p' "$0"
|
||||
exit 0
|
||||
;;
|
||||
*) echo "Unknown arg: $1" >&2; exit 2 ;;
|
||||
esac
|
||||
done
|
||||
|
||||
ts_utc="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
|
||||
hostname_f="$(hostname -f 2>/dev/null || hostname)"
|
||||
kernel="$(uname -r)"
|
||||
uptime_s="$(cut -d. -f1 /proc/uptime)"
|
||||
cpu_n="$(nproc)"
|
||||
loadavg="$(cut -d' ' -f1-3 /proc/loadavg)"
|
||||
load1="$(cut -d' ' -f1 /proc/loadavg)"
|
||||
|
||||
# --- memory ---
|
||||
mem_total_kb="$(awk '/MemTotal:/ {print $2}' /proc/meminfo)"
|
||||
mem_avail_kb="$(awk '/MemAvailable:/ {print $2}' /proc/meminfo)"
|
||||
mem_used_kb=$((mem_total_kb - mem_avail_kb))
|
||||
mem_used_pct=$((mem_used_kb * 100 / mem_total_kb))
|
||||
swap_total_kb="$(awk '/SwapTotal:/ {print $2}' /proc/meminfo)"
|
||||
swap_free_kb="$(awk '/SwapFree:/ {print $2}' /proc/meminfo)"
|
||||
swap_used_kb=$((swap_total_kb - swap_free_kb))
|
||||
|
||||
# --- disk helper: path -> total_k used_k avail_k used_pct ---
|
||||
read_df() {
|
||||
local path="$1"
|
||||
local line total used avail pct_raw
|
||||
line="$(df -P "$path" 2>/dev/null | awk 'NR==2 {print $2,$3,$4,$5}')"
|
||||
if [[ -z "$line" ]]; then
|
||||
echo "0 0 0 0"
|
||||
return
|
||||
fi
|
||||
read -r total used avail pct_raw <<<"$line"
|
||||
echo "$total $used $avail ${pct_raw%%%}"
|
||||
}
|
||||
|
||||
# --- disk root ---
|
||||
read -r disk_total_k disk_used_k disk_avail_k disk_used_pct <<<"$(read_df /)"
|
||||
|
||||
# --- disk Docker root (may be same FS as / or a separate mount) ---
|
||||
docker_disk_same_as_root=1
|
||||
docker_disk_total_k=0
|
||||
docker_disk_used_k=0
|
||||
docker_disk_avail_k=0
|
||||
docker_disk_used_pct=0
|
||||
if [[ -d "$DOCKER_ROOT" ]]; then
|
||||
read -r docker_disk_total_k docker_disk_used_k docker_disk_avail_k docker_disk_used_pct <<<"$(read_df "$DOCKER_ROOT")"
|
||||
root_src="$(findmnt -n -o SOURCE / 2>/dev/null || true)"
|
||||
docker_src="$(findmnt -n -o SOURCE --target "$DOCKER_ROOT" 2>/dev/null || true)"
|
||||
if [[ -n "$root_src" && -n "$docker_src" && "$root_src" != "$docker_src" ]]; then
|
||||
docker_disk_same_as_root=0
|
||||
fi
|
||||
fi
|
||||
|
||||
# --- reboot / updates (lightweight; avoid apt list --upgradable) ---
|
||||
reboot_required=0
|
||||
[[ -f /var/run/reboot-required ]] && reboot_required=1
|
||||
pending_pkgs=0
|
||||
pending_security=0
|
||||
if [[ -x /usr/lib/update-notifier/apt-check ]]; then
|
||||
# stderr: "<total>;<security>"
|
||||
apt_check_out="$(/usr/lib/update-notifier/apt-check 2>&1 || true)"
|
||||
pending_pkgs="${apt_check_out%%;*}"
|
||||
pending_security="${apt_check_out##*;}"
|
||||
elif [[ -f /var/lib/update-notifier/updates-available ]]; then
|
||||
pending_pkgs="$(grep -oE '^[0-9]+' /var/lib/update-notifier/updates-available 2>/dev/null | head -1 || true)"
|
||||
fi
|
||||
[[ "$pending_pkgs" =~ ^[0-9]+$ ]] || pending_pkgs=0
|
||||
[[ "$pending_security" =~ ^[0-9]+$ ]] || pending_security=0
|
||||
|
||||
# --- firewall ---
|
||||
ufw_status="unknown"
|
||||
if command -v ufw >/dev/null 2>&1; then
|
||||
ufw_status="$(ufw status 2>/dev/null | awk 'NR==1 {print tolower($2)}')"
|
||||
fi
|
||||
|
||||
# --- fail2ban ---
|
||||
f2b_active="inactive"
|
||||
f2b_jails=0
|
||||
f2b_banned_now=0
|
||||
f2b_banned_total=0
|
||||
f2b_failed_total=0
|
||||
if command -v fail2ban-client >/dev/null 2>&1; then
|
||||
if systemctl is-active --quiet fail2ban 2>/dev/null; then
|
||||
f2b_active="active"
|
||||
f2b_status="$(fail2ban-client status 2>/dev/null || true)"
|
||||
f2b_jails="$(echo "$f2b_status" | awk -F: '/Jail list/ {gsub(/ /,"",$2); n=split($2,a,","); print n; exit}')"
|
||||
[[ -z "$f2b_jails" ]] && f2b_jails=0
|
||||
if fail2ban-client status sshd >/dev/null 2>&1; then
|
||||
sshd_st="$(fail2ban-client status sshd 2>/dev/null || true)"
|
||||
f2b_banned_now="$(echo "$sshd_st" | awk -F: '/Currently banned/ {gsub(/ /,"",$2); print $2; exit}')"
|
||||
f2b_banned_total="$(echo "$sshd_st" | awk -F: '/Total banned/ {gsub(/ /,"",$2); print $2; exit}')"
|
||||
f2b_failed_total="$(echo "$sshd_st" | awk -F: '/Total failed/ {gsub(/ /,"",$2); print $2; exit}')"
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
f2b_banned_now="${f2b_banned_now:-0}"
|
||||
f2b_banned_total="${f2b_banned_total:-0}"
|
||||
f2b_failed_total="${f2b_failed_total:-0}"
|
||||
|
||||
# --- SSH auth last hour (single journalctl pass) ---
|
||||
ssh_failed_1h=0
|
||||
ssh_invalid_1h=0
|
||||
ssh_accepted_1h=0
|
||||
if command -v journalctl >/dev/null 2>&1; then
|
||||
ssh_journal="$(journalctl -u ssh --since '1 hour ago' --no-pager 2>/dev/null || true)"
|
||||
ssh_failed_1h="$(grep -cE 'Failed password|Failed publickey|Connection closed by authenticating' <<<"$ssh_journal" || true)"
|
||||
ssh_invalid_1h="$(grep -c 'Invalid user' <<<"$ssh_journal" || true)"
|
||||
ssh_accepted_1h="$(grep -c 'Accepted' <<<"$ssh_journal" || true)"
|
||||
fi
|
||||
|
||||
# --- listening public ports ---
|
||||
listen_ports="$(ss -tuln 2>/dev/null | awk '/LISTEN/ && ($5 ~ /\*:|:0\.0\.0\.0:|\[::\]:/) {
|
||||
split($5,a,":"); print a[length(a)]
|
||||
}' | sort -n | uniq | tr '\n' ',' | sed 's/,$//')"
|
||||
|
||||
# --- docker ---
|
||||
docker_running=0
|
||||
docker_unhealthy=0
|
||||
docker_exited=0
|
||||
docker_names_unhealthy=""
|
||||
if command -v docker >/dev/null 2>&1 && docker info >/dev/null 2>&1; then
|
||||
docker_running="$(docker ps -q 2>/dev/null | wc -l | tr -d ' ')"
|
||||
docker_exited="$(docker ps -aq -f status=exited 2>/dev/null | wc -l | tr -d ' ')"
|
||||
while IFS= read -r line; do
|
||||
[[ -z "$line" ]] && continue
|
||||
name="${line%% *}"
|
||||
health="${line##* }"
|
||||
if [[ "$health" == "unhealthy" ]]; then
|
||||
docker_unhealthy=$((docker_unhealthy + 1))
|
||||
docker_names_unhealthy+="${name},"
|
||||
fi
|
||||
done < <(docker ps --format '{{.Names}} {{.Status}}' 2>/dev/null | awk '{
|
||||
h="unknown"
|
||||
if ($0 ~ /unhealthy/) h="unhealthy"
|
||||
else if ($0 ~ /\(healthy\)/) h="healthy"
|
||||
else if ($0 ~ /Up/) h="up"
|
||||
print $1, h
|
||||
}')
|
||||
docker_names_unhealthy="${docker_names_unhealthy%,}"
|
||||
fi
|
||||
|
||||
# --- TLS cert days left (local nginx :443) ---
|
||||
cert_days=-1
|
||||
cert_cn=""
|
||||
if command -v openssl >/dev/null 2>&1; then
|
||||
cert_pem="$(echo | openssl s_client -connect 127.0.0.1:443 -servername "$DOMAIN" 2>/dev/null | openssl x509 2>/dev/null || true)"
|
||||
if [[ -n "$cert_pem" ]]; then
|
||||
cert_cn="$(echo "$cert_pem" | openssl x509 -noout -subject 2>/dev/null | sed 's/.*CN *= *//')"
|
||||
end_date="$(echo "$cert_pem" | openssl x509 -noout -enddate 2>/dev/null | cut -d= -f2)"
|
||||
end_epoch="$(date -d "$end_date" +%s 2>/dev/null || true)"
|
||||
now_epoch="$(date +%s)"
|
||||
if [[ -n "${end_epoch:-}" ]]; then
|
||||
cert_days=$(( (end_epoch - now_epoch) / 86400 ))
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
|
||||
# --- alerts ---
|
||||
alerts=()
|
||||
load_warn="$(awk -v l="$load1" -v n="$cpu_n" -v m="$LOAD_WARN_MULT" 'BEGIN { print (l > n*m) ? 1 : 0 }')"
|
||||
[[ "$load_warn" == "1" ]] && alerts+=("high_load:${load1}>${cpu_n}*${LOAD_WARN_MULT}")
|
||||
[[ "$mem_used_pct" -ge "$MEM_WARN_PCT" ]] && alerts+=("high_memory:${mem_used_pct}%")
|
||||
[[ "$disk_used_pct" -ge "$DISK_CRIT_PCT" ]] && alerts+=("disk_critical:${disk_used_pct}%")
|
||||
[[ "$disk_used_pct" -ge "$DISK_WARN_PCT" && "$disk_used_pct" -lt "$DISK_CRIT_PCT" ]] && alerts+=("disk_warn:${disk_used_pct}%")
|
||||
# Separate Docker mount: independent thresholds (avoid duplicate alerts when same FS)
|
||||
if [[ "$docker_disk_same_as_root" -eq 0 ]]; then
|
||||
[[ "$docker_disk_used_pct" -ge "$DISK_CRIT_PCT" ]] && alerts+=("docker_disk_critical:${docker_disk_used_pct}%")
|
||||
[[ "$docker_disk_used_pct" -ge "$DISK_WARN_PCT" && "$docker_disk_used_pct" -lt "$DISK_CRIT_PCT" ]] && alerts+=("docker_disk_warn:${docker_disk_used_pct}%")
|
||||
fi
|
||||
[[ "$reboot_required" -eq 1 ]] && alerts+=("reboot_required")
|
||||
[[ "$docker_unhealthy" -gt 0 ]] && alerts+=("docker_unhealthy:${docker_names_unhealthy}")
|
||||
[[ "$cert_days" -ge 0 && "$cert_days" -le "$CERT_WARN_DAYS" ]] && alerts+=("cert_expiring:${cert_days}d")
|
||||
[[ "$ssh_failed_1h" -ge "$SSH_FAIL_WARN" ]] && alerts+=("ssh_bruteforce_1h:${ssh_failed_1h}")
|
||||
[[ "$f2b_active" != "active" ]] && alerts+=("fail2ban_inactive")
|
||||
[[ "$ufw_status" != "active" ]] && alerts+=("ufw_inactive")
|
||||
|
||||
alert_count=${#alerts[@]}
|
||||
alert_joined="$(IFS=','; echo "${alerts[*]:-}")"
|
||||
|
||||
# KiB -> MiB (integer; avoids spawning awk)
|
||||
mb() { echo "$((${1:-0} / 1024))"; }
|
||||
|
||||
render_text() {
|
||||
local docker_disk_note="same_fs_as_root"
|
||||
[[ "$docker_disk_same_as_root" -eq 0 ]] && docker_disk_note="separate_mount"
|
||||
cat <<EOF
|
||||
=== HAN VM metrics @ ${ts_utc} ===
|
||||
host: ${hostname_f}
|
||||
kernel: ${kernel}
|
||||
uptime_sec: ${uptime_s}
|
||||
cpu_cores: ${cpu_n}
|
||||
loadavg: ${loadavg}
|
||||
memory: ${mem_used_pct}% used ($(mb "$mem_used_kb")/$(mb "$mem_total_kb") MiB), avail $(mb "$mem_avail_kb") MiB
|
||||
swap: $(mb "$swap_used_kb")/$(mb "$swap_total_kb") MiB
|
||||
disk_root: ${disk_used_pct}% used ($(mb "$disk_used_k")/$(mb "$disk_total_k") MiB), avail $(mb "$disk_avail_k") MiB
|
||||
disk_docker: ${docker_disk_used_pct}% used ($(mb "$docker_disk_used_k")/$(mb "$docker_disk_total_k") MiB), avail $(mb "$docker_disk_avail_k") MiB [${docker_disk_note}: ${DOCKER_ROOT}]
|
||||
reboot_needed: ${reboot_required}
|
||||
apt_upgradable: ${pending_pkgs} (security=${pending_security})
|
||||
ufw: ${ufw_status}
|
||||
listen_ports: ${listen_ports:-none}
|
||||
fail2ban: ${f2b_active}, jails=${f2b_jails}, banned_now=${f2b_banned_now}, banned_total=${f2b_banned_total}, failed_total=${f2b_failed_total}
|
||||
ssh_1h: failed=${ssh_failed_1h}, invalid_user=${ssh_invalid_1h}, accepted=${ssh_accepted_1h}
|
||||
docker: running=${docker_running}, exited=${docker_exited}, unhealthy=${docker_unhealthy}${docker_names_unhealthy:+ ($docker_names_unhealthy)}
|
||||
tls: domain=${DOMAIN}, cn=${cert_cn:-n/a}, days_left=${cert_days}
|
||||
alerts(${alert_count}): ${alert_joined:-none}
|
||||
EOF
|
||||
}
|
||||
|
||||
render_json() {
|
||||
python3 - "$ts_utc" <<'PY' || true
|
||||
import json, os, sys
|
||||
ts = sys.argv[1]
|
||||
data = {
|
||||
"ts": ts,
|
||||
"host": os.environ["M_HOST"],
|
||||
"kernel": os.environ["M_KERNEL"],
|
||||
"uptime_sec": int(os.environ["M_UPTIME"]),
|
||||
"cpu_cores": int(os.environ["M_CPU"]),
|
||||
"loadavg": os.environ["M_LOADAVG"],
|
||||
"load1": float(os.environ["M_LOAD1"]),
|
||||
"memory_used_pct": int(os.environ["M_MEM_PCT"]),
|
||||
"disk_root_used_pct": int(os.environ["M_DISK_PCT"]),
|
||||
"disk_docker": {
|
||||
"path": os.environ["M_D_DISK_PATH"],
|
||||
"used_pct": int(os.environ["M_D_DISK_PCT"]),
|
||||
"same_fs_as_root": os.environ["M_D_DISK_SAME"] == "1",
|
||||
},
|
||||
"reboot_required": os.environ["M_REBOOT"] == "1",
|
||||
"apt_upgradable": int(os.environ["M_APT"]),
|
||||
"apt_security": int(os.environ["M_APT_SEC"]),
|
||||
"ufw": os.environ["M_UFW"],
|
||||
"listen_ports": [p for p in os.environ.get("M_PORTS","").split(",") if p],
|
||||
"fail2ban": {
|
||||
"active": os.environ["M_F2B"] == "active",
|
||||
"jails": int(os.environ["M_F2B_JAILS"]),
|
||||
"banned_now": int(os.environ["M_F2B_NOW"]),
|
||||
"banned_total": int(os.environ["M_F2B_TOT"]),
|
||||
"failed_total": int(os.environ["M_F2B_FAIL"]),
|
||||
},
|
||||
"ssh_1h": {
|
||||
"failed": int(os.environ["M_SSH_FAIL"]),
|
||||
"invalid_user": int(os.environ["M_SSH_INV"]),
|
||||
"accepted": int(os.environ["M_SSH_OK"]),
|
||||
},
|
||||
"docker": {
|
||||
"running": int(os.environ["M_D_RUN"]),
|
||||
"exited": int(os.environ["M_D_EXIT"]),
|
||||
"unhealthy": int(os.environ["M_D_BAD"]),
|
||||
"unhealthy_names": [n for n in os.environ.get("M_D_BAD_N","").split(",") if n],
|
||||
},
|
||||
"tls": {
|
||||
"domain": os.environ["M_DOMAIN"],
|
||||
"cn": os.environ.get("M_CERT_CN") or None,
|
||||
"days_left": int(os.environ["M_CERT_DAYS"]),
|
||||
},
|
||||
"alerts": [a for a in os.environ.get("M_ALERTS","").split(",") if a],
|
||||
}
|
||||
print(json.dumps(data, ensure_ascii=False))
|
||||
PY
|
||||
}
|
||||
|
||||
export M_HOST="$hostname_f" M_KERNEL="$kernel" M_UPTIME="$uptime_s" M_CPU="$cpu_n"
|
||||
export M_LOADAVG="$loadavg" M_LOAD1="$load1" M_MEM_PCT="$mem_used_pct" M_DISK_PCT="$disk_used_pct"
|
||||
export M_D_DISK_PATH="$DOCKER_ROOT" M_D_DISK_PCT="$docker_disk_used_pct" M_D_DISK_SAME="$docker_disk_same_as_root"
|
||||
export M_REBOOT="$reboot_required" M_APT="$pending_pkgs" M_APT_SEC="$pending_security"
|
||||
export M_UFW="$ufw_status" M_PORTS="$listen_ports"
|
||||
export M_F2B="$f2b_active" M_F2B_JAILS="$f2b_jails" M_F2B_NOW="$f2b_banned_now"
|
||||
export M_F2B_TOT="$f2b_banned_total" M_F2B_FAIL="$f2b_failed_total"
|
||||
export M_SSH_FAIL="$ssh_failed_1h" M_SSH_INV="$ssh_invalid_1h" M_SSH_OK="$ssh_accepted_1h"
|
||||
export M_D_RUN="$docker_running" M_D_EXIT="$docker_exited" M_D_BAD="$docker_unhealthy" M_D_BAD_N="$docker_names_unhealthy"
|
||||
export M_DOMAIN="$DOMAIN" M_CERT_CN="$cert_cn" M_CERT_DAYS="$cert_days" M_ALERTS="$alert_joined"
|
||||
|
||||
if [[ "$MODE" == "json" ]]; then
|
||||
out="$(render_json)"
|
||||
else
|
||||
out="$(render_text)"
|
||||
fi
|
||||
|
||||
printf '%s\n' "$out"
|
||||
|
||||
if [[ -n "$LOG_FILE" ]]; then
|
||||
mkdir -p "$(dirname "$LOG_FILE")"
|
||||
printf '%s\n' "$out" >>"$LOG_FILE"
|
||||
fi
|
||||
|
||||
if [[ "$ALERT" -eq 1 && "$alert_count" -gt 0 ]]; then
|
||||
exit 1
|
||||
fi
|
||||
exit 0
|
||||
Reference in New Issue
Block a user