Реализация на отдельных двух машинах с протестированным взаимодействием по проверке сообщений

support&notes/scripts/diagnose-han-chat.sh

@@ -0,0 +1,74 @@
+#!/usr/bin/env bash
+# diagnose-han-chat.sh — запускать из /opt/han-chat/backend
+set -euo pipefail
+cd /opt/han-chat/backend
+CONFIG_FILE="${CONFIG_FILE:-.env}"
+SECRETS_LAUNCHER="${SECRETS_LAUNCHER:-deployment/secrets/han-secrets}"
+if [[ "${HAN_SECRETS_ACTIVE:-0}" != 1 ]]; then
+  [[ -x "$SECRETS_LAUNCHER" ]] || {
+    echo "Secret launcher is required: $SECRETS_LAUNCHER" >&2
+    exit 66
+  }
+  exec "$SECRETS_LAUNCHER" run --config "$CONFIG_FILE" -- "$0" "$@"
+fi
+compose() { docker compose --env-file "$CONFIG_FILE" "$@"; }
+
+echo "=== 1. Статус сервисов ==="
+compose ps api-backend keycloak redis message-safety
+
+echo
+echo "=== 2. Healthcheck api-backend (что именно падает) ==="
+compose exec -T api-backend \
+  python -c "
+import json, urllib.request
+try:
+    r = urllib.request.urlopen('http://127.0.0.1:8000/health/ready', timeout=5)
+    print('HTTP', r.status)
+    print(json.dumps(json.loads(r.read()), indent=2, ensure_ascii=False))
+except urllib.error.HTTPError as e:
+    print('HTTP', e.code)
+    print(json.dumps(json.loads(e.read()), indent=2, ensure_ascii=False))
+"
+
+echo
+echo "=== 3. Keycloak health (management port 9000) ==="
+compose exec -T keycloak bash -c \
+  "exec 3<>/dev/tcp/127.0.0.1/9000 && printf 'GET /auth/health/ready HTTP/1.0\r\n\r\n' >&3 && cat <&3" \
+  2>/dev/null | head -20 || echo "Keycloak health FAILED"
+
+echo
+echo "=== 4. api-backend -> Keycloak (JWKS) ==="
+compose exec -T api-backend \
+  python -c "
+import os, urllib.request, json
+base = os.environ.get('KEYCLOAK_INTERNAL_URL','http://keycloak:8080/auth').rstrip('/')
+realm = os.environ.get('KEYCLOAK_REALM','han-chat')
+url = f'{base}/realms/{realm}/.well-known/openid-configuration'
+try:
+    d = json.loads(urllib.request.urlopen(url, timeout=5).read())
+    print('discovery OK, jwks_uri:', d.get('jwks_uri'))
+except Exception as e:
+    print('discovery FAILED:', e)
+"
+
+echo
+echo "=== 5. Счётчик рестартов ==="
+docker inspect --format '{{.Name}} restarts={{.RestartCount}} started={{.State.StartedAt}} health={{if .State.Health}}{{.State.Health.Status}}{{else}}n/a{{end}}' \
+  han-chat-api-backend-1 han-chat-keycloak-1
+
+echo
+echo "=== 6. Последние ошибки в логах ==="
+echo "--- api-backend ---"
+docker logs --tail 80 han-chat-api-backend-1 2>&1 | grep -iE 'error|failed|exception|jwks|settings|postgres' || echo "(нет совпадений)"
+echo "--- keycloak ---"
+docker logs --tail 80 han-chat-keycloak-1 2>&1 | grep -iE 'error|failed|exception|FATAL|OutOfMemory' || echo "(нет совпадений)"
+
+echo
+echo "=== 7. Память (Keycloak часто падает от OOM) ==="
+free -h
+docker stats --no-stream --format 'table {{.Name}}\t{{.MemUsage}}\t{{.CPUPerc}}' \
+  han-chat-keycloak-1 han-chat-api-backend-1 2>/dev/null || true
+
+echo
+echo "=== 8. Exited контейнеры (в метриках было exited=2) ==="
+docker ps -a --filter status=exited --format 'table {{.Names}}\t{{.Status}}\t{{.Image}}'

support&notes/scripts/han-vm-metrics.sh

@@ -0,0 +1,345 @@
+#!/usr/bin/env bash
+# HAN Chat VM metrics snapshot
+# Host: Ubuntu 24.04 + Docker + UFW + fail2ban
+#
+# Usage:
+#   sudo ./han-vm-metrics.sh              # human report to stdout
+#   sudo ./han-vm-metrics.sh --json       # JSON to stdout
+#   sudo ./han-vm-metrics.sh --alert      # exit 1 if thresholds breached
+#   sudo ./han-vm-metrics.sh --log /var/log/han-vm-metrics.log
+#
+# Cron example (every 5 min):
+#   */5 * * * * root /opt/han-chat/ops/han-vm-metrics.sh --alert --log /var/log/han-vm-metrics.log
+#
+# Disk notes:
+#   Checks both / and /var/lib/docker. If Docker Root Dir is on a separate
+#   mount, its fill level is reported and alerted independently.
+
+set -euo pipefail
+
+MODE="text"
+LOG_FILE=""
+ALERT=0
+DISK_WARN_PCT="${DISK_WARN_PCT:-80}"
+DISK_CRIT_PCT="${DISK_CRIT_PCT:-90}"
+LOAD_WARN_MULT="${LOAD_WARN_MULT:-1.5}"   # vs CPU count
+MEM_WARN_PCT="${MEM_WARN_PCT:-90}"
+CERT_WARN_DAYS="${CERT_WARN_DAYS:-21}"
+SSH_FAIL_WARN="${SSH_FAIL_WARN:-50}"      # failed attempts last hour
+DOMAIN="${METRICS_DOMAIN:-chat.han0107.ru}"
+DOCKER_ROOT="${DOCKER_ROOT:-/var/lib/docker}"
+
+require_value() {
+  local opt="$1" val="${2:-}"
+  if [[ -z "$val" || "$val" == -* ]]; then
+    echo "Error: $opt requires a value" >&2
+    exit 2
+  fi
+}
+
+while [[ $# -gt 0 ]]; do
+  case "$1" in
+    --json) MODE="json"; shift ;;
+    --alert) ALERT=1; shift ;;
+    --log)
+      require_value "$1" "${2:-}"
+      LOG_FILE="$2"
+      shift 2
+      ;;
+    --domain)
+      require_value "$1" "${2:-}"
+      DOMAIN="$2"
+      shift 2
+      ;;
+    -h|--help)
+      sed -n '2,18p' "$0"
+      exit 0
+      ;;
+    *) echo "Unknown arg: $1" >&2; exit 2 ;;
+  esac
+done
+
+ts_utc="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
+hostname_f="$(hostname -f 2>/dev/null || hostname)"
+kernel="$(uname -r)"
+uptime_s="$(cut -d. -f1 /proc/uptime)"
+cpu_n="$(nproc)"
+loadavg="$(cut -d' ' -f1-3 /proc/loadavg)"
+load1="$(cut -d' ' -f1 /proc/loadavg)"
+
+# --- memory ---
+mem_total_kb="$(awk '/MemTotal:/ {print $2}' /proc/meminfo)"
+mem_avail_kb="$(awk '/MemAvailable:/ {print $2}' /proc/meminfo)"
+mem_used_kb=$((mem_total_kb - mem_avail_kb))
+mem_used_pct=$((mem_used_kb * 100 / mem_total_kb))
+swap_total_kb="$(awk '/SwapTotal:/ {print $2}' /proc/meminfo)"
+swap_free_kb="$(awk '/SwapFree:/ {print $2}' /proc/meminfo)"
+swap_used_kb=$((swap_total_kb - swap_free_kb))
+
+# --- disk helper: path -> total_k used_k avail_k used_pct ---
+read_df() {
+  local path="$1"
+  local line total used avail pct_raw
+  line="$(df -P "$path" 2>/dev/null | awk 'NR==2 {print $2,$3,$4,$5}')"
+  if [[ -z "$line" ]]; then
+    echo "0 0 0 0"
+    return
+  fi
+  read -r total used avail pct_raw <<<"$line"
+  echo "$total $used $avail ${pct_raw%%%}"
+}
+
+# --- disk root ---
+read -r disk_total_k disk_used_k disk_avail_k disk_used_pct <<<"$(read_df /)"
+
+# --- disk Docker root (may be same FS as / or a separate mount) ---
+docker_disk_same_as_root=1
+docker_disk_total_k=0
+docker_disk_used_k=0
+docker_disk_avail_k=0
+docker_disk_used_pct=0
+if [[ -d "$DOCKER_ROOT" ]]; then
+  read -r docker_disk_total_k docker_disk_used_k docker_disk_avail_k docker_disk_used_pct <<<"$(read_df "$DOCKER_ROOT")"
+  root_src="$(findmnt -n -o SOURCE / 2>/dev/null || true)"
+  docker_src="$(findmnt -n -o SOURCE --target "$DOCKER_ROOT" 2>/dev/null || true)"
+  if [[ -n "$root_src" && -n "$docker_src" && "$root_src" != "$docker_src" ]]; then
+    docker_disk_same_as_root=0
+  fi
+fi
+
+# --- reboot / updates (lightweight; avoid apt list --upgradable) ---
+reboot_required=0
+[[ -f /var/run/reboot-required ]] && reboot_required=1
+pending_pkgs=0
+pending_security=0
+if [[ -x /usr/lib/update-notifier/apt-check ]]; then
+  # stderr: "<total>;<security>"
+  apt_check_out="$(/usr/lib/update-notifier/apt-check 2>&1 || true)"
+  pending_pkgs="${apt_check_out%%;*}"
+  pending_security="${apt_check_out##*;}"
+elif [[ -f /var/lib/update-notifier/updates-available ]]; then
+  pending_pkgs="$(grep -oE '^[0-9]+' /var/lib/update-notifier/updates-available 2>/dev/null | head -1 || true)"
+fi
+[[ "$pending_pkgs" =~ ^[0-9]+$ ]] || pending_pkgs=0
+[[ "$pending_security" =~ ^[0-9]+$ ]] || pending_security=0
+
+# --- firewall ---
+ufw_status="unknown"
+if command -v ufw >/dev/null 2>&1; then
+  ufw_status="$(ufw status 2>/dev/null | awk 'NR==1 {print tolower($2)}')"
+fi
+
+# --- fail2ban ---
+f2b_active="inactive"
+f2b_jails=0
+f2b_banned_now=0
+f2b_banned_total=0
+f2b_failed_total=0
+if command -v fail2ban-client >/dev/null 2>&1; then
+  if systemctl is-active --quiet fail2ban 2>/dev/null; then
+    f2b_active="active"
+    f2b_status="$(fail2ban-client status 2>/dev/null || true)"
+    f2b_jails="$(echo "$f2b_status" | awk -F: '/Jail list/ {gsub(/ /,"",$2); n=split($2,a,","); print n; exit}')"
+    [[ -z "$f2b_jails" ]] && f2b_jails=0
+    if fail2ban-client status sshd >/dev/null 2>&1; then
+      sshd_st="$(fail2ban-client status sshd 2>/dev/null || true)"
+      f2b_banned_now="$(echo "$sshd_st" | awk -F: '/Currently banned/ {gsub(/ /,"",$2); print $2; exit}')"
+      f2b_banned_total="$(echo "$sshd_st" | awk -F: '/Total banned/ {gsub(/ /,"",$2); print $2; exit}')"
+      f2b_failed_total="$(echo "$sshd_st" | awk -F: '/Total failed/ {gsub(/ /,"",$2); print $2; exit}')"
+    fi
+  fi
+fi
+f2b_banned_now="${f2b_banned_now:-0}"
+f2b_banned_total="${f2b_banned_total:-0}"
+f2b_failed_total="${f2b_failed_total:-0}"
+
+# --- SSH auth last hour (single journalctl pass) ---
+ssh_failed_1h=0
+ssh_invalid_1h=0
+ssh_accepted_1h=0
+if command -v journalctl >/dev/null 2>&1; then
+  ssh_journal="$(journalctl -u ssh --since '1 hour ago' --no-pager 2>/dev/null || true)"
+  ssh_failed_1h="$(grep -cE 'Failed password|Failed publickey|Connection closed by authenticating' <<<"$ssh_journal" || true)"
+  ssh_invalid_1h="$(grep -c 'Invalid user' <<<"$ssh_journal" || true)"
+  ssh_accepted_1h="$(grep -c 'Accepted' <<<"$ssh_journal" || true)"
+fi
+
+# --- listening public ports ---
+listen_ports="$(ss -tuln 2>/dev/null | awk '/LISTEN/ && ($5 ~ /\*:|:0\.0\.0\.0:|\[::\]:/) {
+  split($5,a,":"); print a[length(a)]
+}' | sort -n | uniq | tr '\n' ',' | sed 's/,$//')"
+
+# --- docker ---
+docker_running=0
+docker_unhealthy=0
+docker_exited=0
+docker_names_unhealthy=""
+if command -v docker >/dev/null 2>&1 && docker info >/dev/null 2>&1; then
+  docker_running="$(docker ps -q 2>/dev/null | wc -l | tr -d ' ')"
+  docker_exited="$(docker ps -aq -f status=exited 2>/dev/null | wc -l | tr -d ' ')"
+  while IFS= read -r line; do
+    [[ -z "$line" ]] && continue
+    name="${line%% *}"
+    health="${line##* }"
+    if [[ "$health" == "unhealthy" ]]; then
+      docker_unhealthy=$((docker_unhealthy + 1))
+      docker_names_unhealthy+="${name},"
+    fi
+  done < <(docker ps --format '{{.Names}} {{.Status}}' 2>/dev/null | awk '{
+    h="unknown"
+    if ($0 ~ /unhealthy/) h="unhealthy"
+    else if ($0 ~ /\(healthy\)/) h="healthy"
+    else if ($0 ~ /Up/) h="up"
+    print $1, h
+  }')
+  docker_names_unhealthy="${docker_names_unhealthy%,}"
+fi
+
+# --- TLS cert days left (local nginx :443) ---
+cert_days=-1
+cert_cn=""
+if command -v openssl >/dev/null 2>&1; then
+  cert_pem="$(echo | openssl s_client -connect 127.0.0.1:443 -servername "$DOMAIN" 2>/dev/null | openssl x509 2>/dev/null || true)"
+  if [[ -n "$cert_pem" ]]; then
+    cert_cn="$(echo "$cert_pem" | openssl x509 -noout -subject 2>/dev/null | sed 's/.*CN *= *//')"
+    end_date="$(echo "$cert_pem" | openssl x509 -noout -enddate 2>/dev/null | cut -d= -f2)"
+    end_epoch="$(date -d "$end_date" +%s 2>/dev/null || true)"
+    now_epoch="$(date +%s)"
+    if [[ -n "${end_epoch:-}" ]]; then
+      cert_days=$(( (end_epoch - now_epoch) / 86400 ))
+    fi
+  fi
+fi
+
+# --- alerts ---
+alerts=()
+load_warn="$(awk -v l="$load1" -v n="$cpu_n" -v m="$LOAD_WARN_MULT" 'BEGIN { print (l > n*m) ? 1 : 0 }')"
+[[ "$load_warn" == "1" ]] && alerts+=("high_load:${load1}>${cpu_n}*${LOAD_WARN_MULT}")
+[[ "$mem_used_pct" -ge "$MEM_WARN_PCT" ]] && alerts+=("high_memory:${mem_used_pct}%")
+[[ "$disk_used_pct" -ge "$DISK_CRIT_PCT" ]] && alerts+=("disk_critical:${disk_used_pct}%")
+[[ "$disk_used_pct" -ge "$DISK_WARN_PCT" && "$disk_used_pct" -lt "$DISK_CRIT_PCT" ]] && alerts+=("disk_warn:${disk_used_pct}%")
+# Separate Docker mount: independent thresholds (avoid duplicate alerts when same FS)
+if [[ "$docker_disk_same_as_root" -eq 0 ]]; then
+  [[ "$docker_disk_used_pct" -ge "$DISK_CRIT_PCT" ]] && alerts+=("docker_disk_critical:${docker_disk_used_pct}%")
+  [[ "$docker_disk_used_pct" -ge "$DISK_WARN_PCT" && "$docker_disk_used_pct" -lt "$DISK_CRIT_PCT" ]] && alerts+=("docker_disk_warn:${docker_disk_used_pct}%")
+fi
+[[ "$reboot_required" -eq 1 ]] && alerts+=("reboot_required")
+[[ "$docker_unhealthy" -gt 0 ]] && alerts+=("docker_unhealthy:${docker_names_unhealthy}")
+[[ "$cert_days" -ge 0 && "$cert_days" -le "$CERT_WARN_DAYS" ]] && alerts+=("cert_expiring:${cert_days}d")
+[[ "$ssh_failed_1h" -ge "$SSH_FAIL_WARN" ]] && alerts+=("ssh_bruteforce_1h:${ssh_failed_1h}")
+[[ "$f2b_active" != "active" ]] && alerts+=("fail2ban_inactive")
+[[ "$ufw_status" != "active" ]] && alerts+=("ufw_inactive")
+
+alert_count=${#alerts[@]}
+alert_joined="$(IFS=','; echo "${alerts[*]:-}")"
+
+# KiB -> MiB (integer; avoids spawning awk)
+mb() { echo "$((${1:-0} / 1024))"; }
+
+render_text() {
+  local docker_disk_note="same_fs_as_root"
+  [[ "$docker_disk_same_as_root" -eq 0 ]] && docker_disk_note="separate_mount"
+  cat <<EOF
+=== HAN VM metrics @ ${ts_utc} ===
+host:           ${hostname_f}
+kernel:         ${kernel}
+uptime_sec:     ${uptime_s}
+cpu_cores:      ${cpu_n}
+loadavg:        ${loadavg}
+memory:         ${mem_used_pct}% used ($(mb "$mem_used_kb")/$(mb "$mem_total_kb") MiB), avail $(mb "$mem_avail_kb") MiB
+swap:           $(mb "$swap_used_kb")/$(mb "$swap_total_kb") MiB
+disk_root:      ${disk_used_pct}% used ($(mb "$disk_used_k")/$(mb "$disk_total_k") MiB), avail $(mb "$disk_avail_k") MiB
+disk_docker:    ${docker_disk_used_pct}% used ($(mb "$docker_disk_used_k")/$(mb "$docker_disk_total_k") MiB), avail $(mb "$docker_disk_avail_k") MiB [${docker_disk_note}: ${DOCKER_ROOT}]
+reboot_needed:  ${reboot_required}
+apt_upgradable: ${pending_pkgs} (security=${pending_security})
+ufw:            ${ufw_status}
+listen_ports:   ${listen_ports:-none}
+fail2ban:       ${f2b_active}, jails=${f2b_jails}, banned_now=${f2b_banned_now}, banned_total=${f2b_banned_total}, failed_total=${f2b_failed_total}
+ssh_1h:         failed=${ssh_failed_1h}, invalid_user=${ssh_invalid_1h}, accepted=${ssh_accepted_1h}
+docker:         running=${docker_running}, exited=${docker_exited}, unhealthy=${docker_unhealthy}${docker_names_unhealthy:+ ($docker_names_unhealthy)}
+tls:            domain=${DOMAIN}, cn=${cert_cn:-n/a}, days_left=${cert_days}
+alerts(${alert_count}): ${alert_joined:-none}
+EOF
+}
+
+render_json() {
+  python3 - "$ts_utc" <<'PY' || true
+import json, os, sys
+ts = sys.argv[1]
+data = {
+  "ts": ts,
+  "host": os.environ["M_HOST"],
+  "kernel": os.environ["M_KERNEL"],
+  "uptime_sec": int(os.environ["M_UPTIME"]),
+  "cpu_cores": int(os.environ["M_CPU"]),
+  "loadavg": os.environ["M_LOADAVG"],
+  "load1": float(os.environ["M_LOAD1"]),
+  "memory_used_pct": int(os.environ["M_MEM_PCT"]),
+  "disk_root_used_pct": int(os.environ["M_DISK_PCT"]),
+  "disk_docker": {
+    "path": os.environ["M_D_DISK_PATH"],
+    "used_pct": int(os.environ["M_D_DISK_PCT"]),
+    "same_fs_as_root": os.environ["M_D_DISK_SAME"] == "1",
+  },
+  "reboot_required": os.environ["M_REBOOT"] == "1",
+  "apt_upgradable": int(os.environ["M_APT"]),
+  "apt_security": int(os.environ["M_APT_SEC"]),
+  "ufw": os.environ["M_UFW"],
+  "listen_ports": [p for p in os.environ.get("M_PORTS","").split(",") if p],
+  "fail2ban": {
+    "active": os.environ["M_F2B"] == "active",
+    "jails": int(os.environ["M_F2B_JAILS"]),
+    "banned_now": int(os.environ["M_F2B_NOW"]),
+    "banned_total": int(os.environ["M_F2B_TOT"]),
+    "failed_total": int(os.environ["M_F2B_FAIL"]),
+  },
+  "ssh_1h": {
+    "failed": int(os.environ["M_SSH_FAIL"]),
+    "invalid_user": int(os.environ["M_SSH_INV"]),
+    "accepted": int(os.environ["M_SSH_OK"]),
+  },
+  "docker": {
+    "running": int(os.environ["M_D_RUN"]),
+    "exited": int(os.environ["M_D_EXIT"]),
+    "unhealthy": int(os.environ["M_D_BAD"]),
+    "unhealthy_names": [n for n in os.environ.get("M_D_BAD_N","").split(",") if n],
+  },
+  "tls": {
+    "domain": os.environ["M_DOMAIN"],
+    "cn": os.environ.get("M_CERT_CN") or None,
+    "days_left": int(os.environ["M_CERT_DAYS"]),
+  },
+  "alerts": [a for a in os.environ.get("M_ALERTS","").split(",") if a],
+}
+print(json.dumps(data, ensure_ascii=False))
+PY
+}
+
+export M_HOST="$hostname_f" M_KERNEL="$kernel" M_UPTIME="$uptime_s" M_CPU="$cpu_n"
+export M_LOADAVG="$loadavg" M_LOAD1="$load1" M_MEM_PCT="$mem_used_pct" M_DISK_PCT="$disk_used_pct"
+export M_D_DISK_PATH="$DOCKER_ROOT" M_D_DISK_PCT="$docker_disk_used_pct" M_D_DISK_SAME="$docker_disk_same_as_root"
+export M_REBOOT="$reboot_required" M_APT="$pending_pkgs" M_APT_SEC="$pending_security"
+export M_UFW="$ufw_status" M_PORTS="$listen_ports"
+export M_F2B="$f2b_active" M_F2B_JAILS="$f2b_jails" M_F2B_NOW="$f2b_banned_now"
+export M_F2B_TOT="$f2b_banned_total" M_F2B_FAIL="$f2b_failed_total"
+export M_SSH_FAIL="$ssh_failed_1h" M_SSH_INV="$ssh_invalid_1h" M_SSH_OK="$ssh_accepted_1h"
+export M_D_RUN="$docker_running" M_D_EXIT="$docker_exited" M_D_BAD="$docker_unhealthy" M_D_BAD_N="$docker_names_unhealthy"
+export M_DOMAIN="$DOMAIN" M_CERT_CN="$cert_cn" M_CERT_DAYS="$cert_days" M_ALERTS="$alert_joined"
+
+if [[ "$MODE" == "json" ]]; then
+  out="$(render_json)"
+else
+  out="$(render_text)"
+fi
+
+printf '%s\n' "$out"
+
+if [[ -n "$LOG_FILE" ]]; then
+  mkdir -p "$(dirname "$LOG_FILE")"
+  printf '%s\n' "$out" >>"$LOG_FILE"
+fi
+
+if [[ "$ALERT" -eq 1 && "$alert_count" -gt 0 ]]; then
+  exit 1
+fi
+exit 0

support&notes/scripts/instructions.md

@@ -0,0 +1,53 @@
+# han-vm-metrics.sh
+Установка на ВМ
+sudo mkdir -p /opt/han-chat/ops
+Cкопировать han-vm-metrics.sh на сервер, затем:
+sudo chmod +x /opt/han-chat/ops/han-vm-metrics.sh
+sudo /opt/han-chat/ops/han-vm-metrics.sh
+JSON:
+
+sudo /opt/han-chat/ops/han-vm-metrics.sh --json
+Cron каждые 5 минут + лог + ненулевой exit при проблемах:
+
+echo '*/5 * * * * root /opt/han-chat/ops/han-vm-metrics.sh --alert --log /var/log/han-vm-metrics.log' | sudo tee /etc/cron.d/han-vm-metrics
+Пороги можно менять env: DISK_WARN_PCT, DISK_CRIT_PCT, MEM_WARN_PCT, CERT_WARN_DAYS, SSH_FAIL_WARN.
+
+# diagnose-han-chat.sh
+
+
+# КОнтроль места
+docker images --format 'table {{.Repository}}\t{{.Tag}}\t{{.Size}}\t{{.ID}}\t{{.CreatedSince}}'
+docker builder du
+
+# Очистка build cache
+docker builder prune -af
+
+# Cron раз в сутки (03:15) + лог
+echo '15 3 * * * root /usr/bin/docker builder prune -af >> /var/log/docker-builder-prune.log 2>&1' | sudo tee /etc/cron.d/docker-builder-prune
+sudo chmod 644 /etc/cron.d/docker-builder-prune
+
+Логи
+cat /etc/cron.d/docker-builder-prune
+tail -n 20 /var/log/docker-builder-prune.log
+
+# Проверка места
+df -h /
+docker system df
+
+
+# Обновление секретов
+
+### Как проверить, что sync реально отработал
+
+```sh
+# 1) Когда unit реально выполнялся
+systemctl show han-secrets-vm2.service -p ActiveState -p SubState -p Result -p ExecMainStartTimestamp -p ExecMainStatus
+
+# 2) Журнал последней попытки
+journalctl -u han-secrets-vm2.service -n 50 --no-pager
+
+# 3) Принудительно пересинхронизировать
+systemctl restart han-secrets-vm2.service
+echo exit:$?
+systemctl show han-secrets-vm2.service -p ExecMainStartTimestamp -p Result -p ExecMainStatus
+```

support&notes/scripts/message_lost_diagnostic.md

@@ -0,0 +1,36 @@
+cd /opt/han-chat/backend
+
+echo "=== CONTAINERS ==="
+docker compose --env-file .env ps
+
+echo "=== BITRIX STATUS ==="
+docker compose --env-file .env exec -T api-backend python - <<'PY'
+import json
+import os
+import urllib.request
+
+request = urllib.request.Request(
+    os.environ["BITRIX_LOCAL_APP_BASE_URL"] + "/internal/openlines/v1/status",
+    headers={
+        "Authorization": "Bearer "
+        + os.environ["BITRIX_LOCAL_APP_INTERNAL_TOKEN"]
+    },
+)
+print(json.dumps(
+    json.load(urllib.request.urlopen(request, timeout=10)),
+    ensure_ascii=False,
+    indent=2,
+))
+PY
+
+echo "=== BITRIX LOGS ==="
+docker compose --env-file .env logs \
+  --since=15m --timestamps bitrix-local-app
+
+echo "=== API LOGS ==="
+docker compose --env-file .env logs \
+  --since=15m --timestamps api-backend
+
+echo "=== NGINX LOGS ==="
+docker compose --env-file .env logs \
+  --since=15m --timestamps nginx

support&notes/scripts/pg_connection_test.sh

@@ -0,0 +1,36 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+# 1. Безопасная загрузка .env: убираем комментарии, удаляем \r (Windows-переносы) и делаем export
+export $(grep -v '^#' .env | sed 's/\r$//' | xargs)
+
+echo "=== 1. Параметры из .env ==="
+grep -E '^(HAN_PG_HOST|HAN_PG_PORT|KEYCLOAK_DB_)' .env | grep -v PASSWORD || echo "Переменные не найдены!"
+
+# 2. Проверка TCP (используем переменные, а не хардкод)
+echo -e "\n=== 2. Проверка TCP (${HAN_PG_HOST}:${HAN_PG_PORT}) ==="
+if command -v nc &> /dev/null; then
+    nc -zv "${HAN_PG_HOST}" "${HAN_PG_PORT}" || echo "❌ Не удалось подключиться по TCP"
+else
+    echo "⚠️ Утилита nc не найдена, пропускаем проверку TCP"
+fi
+
+# 3. Проверка подключения psql
+echo -e "\n=== 3. Проверка подключения к БД ==="
+
+# ВАЖНО: Передаем пароль через переменную окружения, чтобы скрипт не зависал!
+# Если KEYCLOAK_DB_PASSWORD пуст, psql все равно спросит его вручную.
+export PGPASSWORD="${KEYCLOAK_DB_PASSWORD:-}"
+
+# Проверяем, установлен ли psql
+if ! command -v psql &> /dev/null; then
+    echo "❌ Утилита psql не найдена на хост-машине. Установите postgresql-client."
+    exit 1
+fi
+
+# Выполняем запрос. Используем переменную порта и добавляем fallback для sslrootcert, если он не задан
+psql "host=${HAN_PG_HOST} port=${HAN_PG_PORT} dbname=han_chat user=${KEYCLOAK_DB_USERNAME} sslmode=verify-full sslrootcert=${PG_CA_HOST_PATH:-}" \
+  -c "SELECT current_user, current_schema(); SHOW search_path;"
+
+# Очищаем пароль из окружения после выполнения (хорошая практика)
+unset PGPASSWORD

support&notes/scripts/send_sms.md

@@ -0,0 +1,202 @@
+# Отправка СМС
+
+## На ВМ выполните:
+
+cd /opt/han-chat/backend
+umask 077
+
+REQUEST_FILE=$(mktemp)
+read -r -p "Тестовый номер в E.164 (+79...): " TEST_PHONE
+python3 - "$REQUEST_FILE" "$TEST_PHONE" <<'PY'
+import json
+import secrets
+import sys
+import uuid
+
+challenge_id = str(uuid.uuid4())
+payload = {
+    "idempotency_key": f"ops:smoke:{challenge_id}",
+    "template_code": "auth_otp",
+    "locale": "ru",
+    "phone_e164": sys.argv[2],
+    "substitutions": {
+        "code": f"{secrets.randbelow(1000000):06d}",
+        "ttl_min": "1",
+    },
+    "customer_ref": challenge_id,
+    "message_ttl_sec": 60,
+}
+
+with open(sys.argv[1], "w", encoding="utf-8") as file:
+    json.dump(payload, file, ensure_ascii=False)
+PY
+
+
+## Создайте функцию отправки:
+
+send_sms_smoke() {
+  deployment/secrets/han-secrets run --config .env -- \
+  docker compose --env-file .env --profile ops run --rm --no-deps \
+    --user 0:0 \
+    --entrypoint sh \
+    -v /run/han-chat/secrets/SMS_SERVICE_TOKEN:/run/secrets/sms_service_token:ro \
+    -v "$REQUEST_FILE:/tmp/sms-request.json:ro" \
+    toolbox -ec '
+      umask 077
+      printf "Authorization: Bearer %s\n" \
+        "$(cat /run/secrets/sms_service_token)" > /tmp/auth-header
+      curl -sS \
+        -w "\nHTTP %{http_code}\n" \
+        -X POST \
+        -H @/tmp/auth-header \
+        -H "Content-Type: application/json" \
+        -H "X-Request-ID: ops-sms-smoke" \
+        --data-binary @/tmp/sms-request.json \
+        http://sms-service:8080/internal/sms/v1/send
+    '
+}
+
+## Отправка:
+send_sms_smoke
+
+Ожидается:
+HTTP 202 и JSON с sms_message_id.
+
+## Проверьте журнал:
+SELECT
+    id,
+    phone_masked,
+    send_status,
+    delivery_status,
+    provider_message_id,
+    provider_error_code,
+    attempt_count,
+    created_at
+FROM sms.sms_outbound_message
+ORDER BY created_at DESC
+LIMIT 5;
+
+## После проверки удалите секретные данные:
+shred -u "$REQUEST_FILE" 2>/dev/null || rm -f "$REQUEST_FILE"
+unset TEST_PHONE REQUEST_FILE
+
+# Тесты
+
+Выполняйте на ВМ из `/opt/han-chat/backend`.
+
+### 1. Проверить запрет публичного internal API
+
+```bash
+PUBLIC_WEB_URL=$(python3 - <<'PY'
+from pathlib import Path
+for line in Path(".env").read_text().splitlines():
+    if line.startswith("PUBLIC_WEB_URL="):
+        print(line.split("=", 1)[1].strip().strip("\"'"))
+        break
+PY
+)
+
+curl -sS -o /dev/null -w 'HTTP %{http_code}\n' \
+  "$PUBLIC_WEB_URL/internal/sms/v1/messages/00000000-0000-0000-0000-000000000000"
+```
+
+Ожидается:
+
+```text
+HTTP 404
+```
+
+### 2. Проверить callback с неправильного IP
+
+```bash
+curl -sS -o /dev/null -w 'HTTP %{http_code}\n' \
+  -X POST \
+  -H 'Content-Type: application/json' \
+  --data '[]' \
+  "$PUBLIC_WEB_URL/callbacks/idgtl/sms"
+```
+
+Ожидается:
+
+```text
+HTTP 403
+```
+
+Заголовок `X-Forwarded-For` не должен позволять обойти ограничение.
+
+### 3. Проверить Basic auth внутри Docker-сети
+
+Credentials получает только дочерний процесс через secret launcher.
+
+Неверные credentials:
+
+```bash
+deployment/secrets/han-secrets run --config .env -- \
+docker compose --env-file .env --profile ops run --rm --no-deps \
+  --entrypoint sh toolbox -ec '
+    curl -sS -o /dev/null -w "HTTP %{http_code}\n" \
+      -u invalid:invalid \
+      -H "Content-Type: application/json" \
+      --data "[]" \
+      http://sms-service:8080/callbacks/idgtl/sms
+  '
+```
+
+Ожидается `HTTP 401`.
+
+Правильные credentials:
+
+```bash
+deployment/secrets/han-secrets run --config .env -- \
+docker compose --env-file .env --profile ops run --rm --no-deps \
+  --entrypoint sh \
+  -v /run/han-chat/secrets/IDGTL_SMS_CALLBACK_USERNAME:/run/secrets/callback_user:ro \
+  -v /run/han-chat/secrets/IDGTL_SMS_CALLBACK_PASSWORD:/run/secrets/callback_password:ro \
+  toolbox -ec '
+    umask 077
+    {
+      printf "user = \"%s:%s\"\n" \
+        "$(cat /run/secrets/callback_user)" \
+        "$(cat /run/secrets/callback_password)"
+    } > /tmp/curl-auth.conf
+    curl -sS -o /dev/null -w "HTTP %{http_code}\n" \
+      --config /tmp/curl-auth.conf \
+      -H "Content-Type: application/json" \
+      --data "[]" \
+      http://sms-service:8080/callbacks/idgtl/sms
+  '
+```
+
+Ожидается `HTTP 422`: авторизация прошла, но пустой callback-массив невалиден.
+
+### 4. Проверить реальный callback Direct
+
+После тестовой SMS:
+
+```sql
+SELECT
+    id,
+    send_status,
+    delivery_status,
+    provider_message_id,
+    callback_last_at,
+    sent_at,
+    delivered_at
+FROM sms.sms_outbound_message
+ORDER BY created_at DESC
+LIMIT 5;
+```
+
+Успешный реальный callback подтверждается:
+
+- `callback_last_at IS NOT NULL`;
+- `delivery_status = sent` или `delivered`;
+- заполняются `sent_at`/`delivered_at`.
+
+Дополнительно:
+
+```bash
+docker compose --env-file .env logs --since=30m nginx sms-service
+```
+
+Для callback должен быть ответ `204`. Только реальный запрос Direct может полноценно подтвердить IP allowlist.