Добавлен OTLP-провайдер, реализовано отбрасывание метрик и трейсов в observability + добавлен перезапуск nginx при пересборке контейнеров (ошибка, когда докер меняет адреса сервисов)

codebase/backend/observability/docker-compose.yml

@@ -18,12 +18,13 @@ services:
       APP_ENV: ${APP_ENV:-production-like}
       RELEASE_VERSION: ${RELEASE_VERSION:-unknown}
       OTEL_REMOTE_ENDPOINT: ${OTEL_REMOTE_ENDPOINT}
-      OTEL_REMOTE_AUTH_HEADER: ${OTEL_REMOTE_AUTH_HEADER}
+      OTEL_REMOTE_AUTH_HEADER: ${OTEL_REMOTE_AUTH_HEADER:-}
+      OTEL_REMOTE_TLS_INSECURE: ${OTEL_REMOTE_TLS_INSECURE:-false}
     expose: ["4317", "4318", "13133", "8888"]
     volumes:
       - ./otel-collector.yaml:/etc/otelcol/config.yaml:ro
       - otel-queue:/var/lib/otelcol/queue
-    networks: [observability, backend, egress]
+    networks: [observability, egress]
     depends_on:
       otel-queue-init: {condition: service_completed_successfully}
     healthcheck:

codebase/backend/observability/otel-collector.yaml

@@ -16,6 +16,33 @@ receivers:
         - job_name: otel-collector
           static_configs:
             - targets: ["127.0.0.1:8888"]
+        - job_name: sms-service
+          scrape_interval: 15s
+          scrape_timeout: 10s
+          metrics_path: /metrics
+          static_configs:
+            - targets: ["sms-service:8080"]
+              labels: {service_name: sms-service}
+        - job_name: sms-worker
+          scrape_interval: 15s
+          scrape_timeout: 10s
+          metrics_path: /metrics
+          static_configs:
+            - targets: ["sms-worker:9464"]
+              labels: {service_name: sms-worker}
+        - job_name: keycloak
+          scrape_interval: 30s
+          scrape_timeout: 15s
+          metrics_path: /auth/metrics
+          static_configs:
+            - targets: ["keycloak:9000"]
+              labels: {service_name: keycloak}
+          metric_relabel_configs:
+            - action: keep
+              source_labels: [__name__]
+              regex: "(keycloak_.*|jvm_.*|process_.*|system_.*|agroal_.*)"
+            - action: labeldrop
+              regex: "(client_id|uri|session_id|user_id|phone|email)"
 
 processors:
   memory_limiter:
@@ -33,19 +60,37 @@ processors:
       - {key: http.request.header.cookie, action: delete}
       - {key: http.response.header.set-cookie, action: delete}
       - {key: url.query, action: delete}
+      - {key: url.full, action: delete}
+      - {key: http.target, action: delete}
+      - {key: http.request.body, action: delete}
       - {key: db.statement, action: delete}
+      - {key: db.query.text, action: delete}
       - {key: enduser.id, action: delete}
       - {key: user.phone, action: delete}
       - {key: user.email, action: delete}
       - {key: messaging.message.body, action: delete}
+      - {key: aws.s3.key, action: delete}
+      - {key: s3.object.key, action: delete}
   filter/noise:
     error_mode: ignore
     traces:
       span:
         - 'attributes["http.route"] == "/health/live"'
         - 'attributes["http.route"] == "/nginx-health/live"'
-  probabilistic_sampler:
-    sampling_percentage: 10
+  tail_sampling:
+    decision_wait: 10s
+    num_traces: 10000
+    expected_new_traces_per_sec: 50
+    policies:
+      - name: errors
+        type: status_code
+        status_code: {status_codes: [ERROR]}
+      - name: slow
+        type: latency
+        latency: {threshold_ms: 2000}
+      - name: successful-sample
+        type: probabilistic
+        probabilistic: {sampling_percentage: 10}
   batch:
     timeout: 5s
     send_batch_size: 1024
@@ -57,7 +102,7 @@ exporters:
     headers:
       authorization: "${env:OTEL_REMOTE_AUTH_HEADER}"
     tls:
-      insecure: false
+      insecure: "${env:OTEL_REMOTE_TLS_INSECURE}"
     sending_queue:
       enabled: true
       num_consumers: 4
@@ -74,7 +119,7 @@ service:
   pipelines:
     traces:
       receivers: [otlp]
-      processors: [memory_limiter, resource/common, attributes/redact, filter/noise, probabilistic_sampler, batch]
+      processors: [memory_limiter, resource/common, attributes/redact, filter/noise, tail_sampling, batch]
       exporters: [otlp/remote]
     metrics:
       receivers: [otlp, prometheus]