ВМ1: реализован сбор логов
This commit is contained in:
mi
@@ -27,6 +27,8 @@ HARDEN_SSH="${HARDEN_SSH:-false}"
|
||||
LOCK_ACCOUNT_PASSWORDS="${LOCK_ACCOUNT_PASSWORDS:-true}"
|
||||
RESET_UFW="${RESET_UFW:-true}"
|
||||
SKIP_APT_UPGRADE="${SKIP_APT_UPGRADE:-false}"
|
||||
OTEL_HOST_COLLECTOR_VERSION="${OTEL_HOST_COLLECTOR_VERSION:-}"
|
||||
OTEL_HOST_COLLECTOR_SHA256="${OTEL_HOST_COLLECTOR_SHA256:-}"
|
||||
LOG_FILE="${LOG_FILE:-/var/log/han-chat-vm1-setup.log}"
|
||||
|
||||
log() {
|
||||
@@ -120,6 +122,13 @@ configure_time() {
|
||||
timedatectl set-ntp true
|
||||
}
|
||||
|
||||
configure_journald() {
|
||||
step "Persistent journald для host telemetry"
|
||||
install -d -m 2755 -o root -g systemd-journal /var/log/journal
|
||||
systemd-tmpfiles --create --prefix /var/log/journal
|
||||
journalctl --flush
|
||||
}
|
||||
|
||||
install_authorized_key() {
|
||||
local user=$1 source=$2 target="/home/${1}/.ssh/authorized_keys"
|
||||
[[ -f "$source" && ! -L "$source" ]] || die "Не найден обычный key file ${source}"
|
||||
@@ -422,7 +431,7 @@ install_deploy_sudoers() {
|
||||
step "Exact sudoers для deploy"
|
||||
cat >/etc/sudoers.d/deploy <<'EOF'
|
||||
Cmnd_Alias HAN_VM1_UNITS = /usr/bin/systemctl start han-secrets@production.service, /usr/bin/systemctl restart han-secrets@production.service, /usr/bin/systemctl start han-stack@production.service, /usr/bin/systemctl restart han-stack@production.service, /usr/bin/systemctl stop han-stack@production.service
|
||||
Cmnd_Alias HAN_VM1_STATUS = /usr/bin/systemctl --no-pager status han-secrets@production.service, /usr/bin/systemctl --no-pager status han-stack@production.service, /usr/bin/journalctl --no-pager -u han-secrets@production.service, /usr/bin/journalctl --no-pager -u han-stack@production.service
|
||||
Cmnd_Alias HAN_VM1_STATUS = /usr/bin/systemctl --no-pager status han-secrets@production.service, /usr/bin/systemctl --no-pager status han-stack@production.service, /usr/bin/systemctl --no-pager status han-host-otel-collector@production.service, /usr/bin/journalctl --no-pager -u han-secrets@production.service, /usr/bin/journalctl --no-pager -u han-stack@production.service, /usr/bin/journalctl --no-pager -u han-host-otel-collector@production.service
|
||||
deploy ALL=(root) NOPASSWD: HAN_VM1_UNITS, HAN_VM1_STATUS
|
||||
EOF
|
||||
chmod 0440 /etc/sudoers.d/deploy
|
||||
@@ -447,6 +456,15 @@ install_release_helpers_if_possible() {
|
||||
[[ -x "${deployment}/scripts/tls-deploy-hook.sh" ]] || die "TLS hook не executable"
|
||||
[[ -x "${source_dir}/han-compose" ]] || die "han-compose не executable"
|
||||
[[ -x "${source_dir}/han-secrets" ]] || die "han-secrets не executable"
|
||||
[[ -f "${deployment}/han-host-otel-collector@.service" ]] \
|
||||
|| die "han-host-otel-collector@.service отсутствует"
|
||||
[[ -f "${deployment}/observability/otel-host-collector.yaml" ]] \
|
||||
|| die "otel-host-collector.yaml отсутствует"
|
||||
[[ "$OTEL_HOST_COLLECTOR_VERSION" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]] \
|
||||
|| die "Задайте OTEL_HOST_COLLECTOR_VERSION=x.y.z"
|
||||
[[ "$OTEL_HOST_COLLECTOR_SHA256" =~ ^[a-f0-9]{64}$ ]] \
|
||||
|| die "Задайте проверенный OTEL_HOST_COLLECTOR_SHA256"
|
||||
[[ "$(uname -m)" == x86_64 ]] || die "Host Collector artifact рассчитан на amd64"
|
||||
if getent group "$tls_group" >/dev/null; then
|
||||
[[ "$(getent group "$tls_group" | cut -d: -f3)" == "$tls_gid" ]] \
|
||||
|| die "han-nginx-tls имеет неожиданный GID"
|
||||
@@ -472,6 +490,27 @@ install_release_helpers_if_possible() {
|
||||
install -m 0644 -o root -g root \
|
||||
"${deployment}/han-stack@.service" \
|
||||
/etc/systemd/system/han-stack@.service
|
||||
local otel_archive otel_tmp otel_url
|
||||
otel_archive="$(mktemp)"
|
||||
otel_tmp="$(mktemp -d)"
|
||||
otel_url="https://github.com/open-telemetry/opentelemetry-collector-releases/releases/download/v${OTEL_HOST_COLLECTOR_VERSION}/otelcol-contrib_${OTEL_HOST_COLLECTOR_VERSION}_linux_amd64.tar.gz"
|
||||
curl --fail --location --proto '=https' --tlsv1.2 \
|
||||
--output "$otel_archive" "$otel_url"
|
||||
printf '%s %s\n' "$OTEL_HOST_COLLECTOR_SHA256" "$otel_archive" | sha256sum -c -
|
||||
tar -xzf "$otel_archive" -C "$otel_tmp" otelcol-contrib
|
||||
install -m 0755 -o root -g root "$otel_tmp/otelcol-contrib" /usr/local/bin/otelcol-contrib
|
||||
install -d -m 0755 -o root -g root /usr/local/share/han-otel
|
||||
sha256sum /usr/local/bin/otelcol-contrib \
|
||||
>/usr/local/share/han-otel/otelcol-contrib.sha256
|
||||
printf '%s\n' "$OTEL_HOST_COLLECTOR_VERSION" \
|
||||
>/usr/local/share/han-otel/otelcol-contrib.version
|
||||
chmod 0644 /usr/local/share/han-otel/otelcol-contrib.sha256 \
|
||||
/usr/local/share/han-otel/otelcol-contrib.version
|
||||
rm -rf "$otel_archive" "$otel_tmp"
|
||||
install -d -m 0700 -o root -g root /var/lib/han-otel/host-collector
|
||||
install -m 0644 -o root -g root \
|
||||
"${deployment}/han-host-otel-collector@.service" \
|
||||
/etc/systemd/system/han-host-otel-collector@.service
|
||||
install -d -m 0755 -o root -g root /etc/letsencrypt/renewal-hooks/deploy
|
||||
install -m 0755 -o root -g root \
|
||||
"${deployment}/scripts/tls-deploy-hook.sh" \
|
||||
@@ -481,6 +520,7 @@ install_release_helpers_if_possible() {
|
||||
/usr/local/share/doc/han-secrets/SELECTEL_RUNBOOK.ru.md
|
||||
systemctl daemon-reload
|
||||
systemctl enable certbot.timer
|
||||
systemctl enable han-host-otel-collector@production.service
|
||||
log "Helpers установлены; application units не запущены"
|
||||
}
|
||||
|
||||
@@ -539,6 +579,7 @@ main() {
|
||||
check_os
|
||||
update_system
|
||||
configure_time
|
||||
configure_journald
|
||||
create_host_roles
|
||||
configure_account_passwords
|
||||
configure_layout
|
||||
|
||||
@@ -16,11 +16,38 @@ compose() { docker compose --env-file "$CONFIG_FILE" "$@"; }
|
||||
|
||||
NETWORK="${OBSERVABILITY_NETWORK:-han-chat-observability}"
|
||||
COLLECTOR_SERVICE="${COLLECTOR_SERVICE:-otel-collector}"
|
||||
TELEMETRYGEN_IMAGE="${TELEMETRYGEN_IMAGE:-}"
|
||||
errors=0
|
||||
|
||||
ok() { printf 'OK %s\n' "$*"; }
|
||||
fail() { printf 'FAIL %s\n' "$*" >&2; errors=$((errors + 1)); }
|
||||
|
||||
if systemctl is-active --quiet han-host-otel-collector@production.service; then
|
||||
ok "Host Collector service is running"
|
||||
else
|
||||
fail "Host Collector service is not running"
|
||||
fi
|
||||
|
||||
if /usr/local/bin/otelcol-contrib validate \
|
||||
--config=deployment/observability/otel-host-collector.yaml >/dev/null 2>&1; then
|
||||
ok "Host Collector configuration is valid"
|
||||
else
|
||||
fail "Host Collector configuration is invalid"
|
||||
fi
|
||||
|
||||
host_bad_logs="$(
|
||||
journalctl --since=-10min --no-pager \
|
||||
-u han-host-otel-collector@production.service 2>&1 |
|
||||
grep -Ei 'queue is full|permission denied|connection refused|tls:|Unauthenticated|Permanent error' ||
|
||||
true
|
||||
)"
|
||||
if [[ -z "$host_bad_logs" ]]; then
|
||||
ok "No host Collector errors in last 10 minutes"
|
||||
else
|
||||
fail "Host Collector reports read/export/queue errors"
|
||||
printf '%s\n' "$host_bad_logs" >&2
|
||||
fi
|
||||
|
||||
collector_id="$(compose ps -q "$COLLECTOR_SERVICE" 2>/dev/null || true)"
|
||||
if [[ -n "$collector_id" ]] &&
|
||||
[[ "$(docker inspect --format '{{.State.Status}}' "$collector_id")" == running ]]; then
|
||||
@@ -43,12 +70,16 @@ PY
|
||||
fi
|
||||
done
|
||||
|
||||
docker run --rm --network "$NETWORK" \
|
||||
ghcr.io/open-telemetry/opentelemetry-collector-contrib/telemetrygen:latest \
|
||||
if [[ ! "$TELEMETRYGEN_IMAGE" =~ @sha256:[a-f0-9]{64}$ ]]; then
|
||||
fail "TELEMETRYGEN_IMAGE must be pinned by sha256 digest"
|
||||
elif docker run --rm --network "$NETWORK" \
|
||||
"$TELEMETRYGEN_IMAGE" \
|
||||
traces --otlp-endpoint otel-collector:4317 --otlp-insecure \
|
||||
--service han-chat-e2e-canary --traces 100 --rate 20 >/dev/null \
|
||||
&& ok "100 canary traces submitted" \
|
||||
|| fail "telemetrygen failed"
|
||||
--service han-chat-e2e-canary --traces 100 --rate 20 >/dev/null; then
|
||||
ok "100 canary traces submitted"
|
||||
else
|
||||
fail "telemetrygen failed"
|
||||
fi
|
||||
|
||||
bad_logs="$(
|
||||
compose logs --since=10m "$COLLECTOR_SERVICE" 2>&1 |
|
||||
@@ -72,4 +103,12 @@ cat <<'EOF'
|
||||
service.namespace = han-chat
|
||||
Затем выполните synthetic API request и проверьте общий trace между
|
||||
api-backend и dependency spans, service.version и deployment.environment.
|
||||
В Logs проверьте:
|
||||
host.name = <VM1 HOST_NAME>
|
||||
service.name IN (nginx, keycloak, redis, api-backend, sms-service,
|
||||
sms-worker, bitrix-local-app)
|
||||
У canary application log trace_id/span_id должны открывать соответствующий
|
||||
span. Python events должны встречаться один раз, unknown-container и
|
||||
otel-host-collector отсутствовать. Fake token/PII marker не должен находиться
|
||||
ни в logs, ни в traces.
|
||||
EOF
|
||||
|
||||
Reference in New Issue
Block a user