#!/usr/bin/env bash
set -Eeuo pipefail
cd "$(dirname "$0")/../.."
CONFIG_FILE="${CONFIG_FILE:-.env}"
SECRETS_LAUNCHER="${SECRETS_LAUNCHER:-deployment/secrets/han-secrets}"

if [[ "${HAN_SECRETS_ACTIVE:-0}" != 1 ]]; then
  [[ -x "$SECRETS_LAUNCHER" ]] || {
    echo "Secret launcher is required: $SECRETS_LAUNCHER" >&2
    exit 66
  }
  exec "$SECRETS_LAUNCHER" run --config "$CONFIG_FILE" -- "$0" "$@"
fi

compose() { docker compose --env-file "$CONFIG_FILE" "$@"; }

NETWORK="${OBSERVABILITY_NETWORK:-han-chat-observability}"
COLLECTOR_SERVICE="${COLLECTOR_SERVICE:-otel-collector}"
TELEMETRYGEN_IMAGE="${TELEMETRYGEN_IMAGE:-}"
errors=0

ok() { printf 'OK   %s\n' "$*"; }
fail() { printf 'FAIL %s\n' "$*" >&2; errors=$((errors + 1)); }

if systemctl is-active --quiet han-host-otel-collector@production.service; then
  ok "Host Collector service is running"
else
  fail "Host Collector service is not running"
fi

if /usr/local/bin/otelcol-contrib validate \
  --config=deployment/observability/otel-host-collector.yaml >/dev/null 2>&1; then
  ok "Host Collector configuration is valid"
else
  fail "Host Collector configuration is invalid"
fi

host_bad_logs="$(
  journalctl --since=-10min --no-pager \
    -u han-host-otel-collector@production.service 2>&1 |
    grep -Ei 'queue is full|permission denied|connection refused|tls:|Unauthenticated|Permanent error' ||
    true
)"
if [[ -z "$host_bad_logs" ]]; then
  ok "No host Collector errors in last 10 minutes"
else
  fail "Host Collector reports read/export/queue errors"
  printf '%s\n' "$host_bad_logs" >&2
fi

collector_id="$(compose ps -q "$COLLECTOR_SERVICE" 2>/dev/null || true)"
if [[ -n "$collector_id" ]] &&
  [[ "$(docker inspect --format '{{.State.Status}}' "$collector_id")" == running ]]; then
  ok "Collector service is running"
else
  fail "Collector service '$COLLECTOR_SERVICE' is not running"
fi

for service in sms-service sms-worker; do
  if compose exec -T "$service" python - <<'PY' >/dev/null 2>&1
import os
import urllib.request
port = os.environ.get("SMS_METRICS_PORT", "9464") if "worker" in os.environ.get("OTEL_SERVICE_NAME", "") else "8080"
urllib.request.urlopen(f"http://127.0.0.1:{port}/metrics", timeout=3).read(1024)
PY
  then
    ok "${service} metrics endpoint"
  else
    fail "${service} metrics endpoint"
  fi
done

if [[ ! "$TELEMETRYGEN_IMAGE" =~ @sha256:[a-f0-9]{64}$ ]]; then
  fail "TELEMETRYGEN_IMAGE must be pinned by sha256 digest"
elif docker run --rm --network "$NETWORK" \
  "$TELEMETRYGEN_IMAGE" \
  traces --otlp-endpoint otel-collector:4317 --otlp-insecure \
  --service han-chat-e2e-canary --traces 100 --rate 20 >/dev/null; then
  ok "100 canary traces submitted"
else
  fail "telemetrygen failed"
fi

bad_logs="$(
  compose logs --since=10m "$COLLECTOR_SERVICE" 2>&1 |
    grep -Ei 'queue is full|connection refused|tls:|Unauthenticated|Permanent error' || true
)"
if [[ -z "$bad_logs" ]]; then
  ok "No exporter/queue errors in last 10 minutes"
else
  fail "Collector reports exporter/queue errors"
  printf '%s\n' "$bad_logs" >&2
fi

if ((errors)); then
  printf 'Observability verification failed: %d check(s)\n' "$errors" >&2
  exit 1
fi

cat <<'EOF'
Локальный канал исправен. В SigNoz проверьте за последние 15 минут:
  service.name = han-chat-e2e-canary
  service.namespace = han-chat
Затем выполните synthetic API request и проверьте общий trace между
api-backend и dependency spans, service.version и deployment.environment.
В Logs проверьте:
  host.name = <VM1 HOST_NAME>
  service.name IN (nginx, keycloak, redis, api-backend, sms-service,
                   sms-worker, bitrix-local-app)
У canary application log trace_id/span_id должны открывать соответствующий
span. Python events должны встречаться один раз, unknown-container и
otel-host-collector отсутствовать. Fake token/PII marker не должен находиться
ни в logs, ни в traces.
EOF