Добавлен OTLP-провайдер, реализовано отбрасывание метрик и трейсов в observability + добавлен перезапуск nginx при пересборке контейнеров (ошибка, когда докер меняет адреса сервисов)

codebase/backend/api-backend/app/main.py

@@ -50,6 +50,7 @@ from app.integrations import (
     S3Client,
     SafetyClient,
 )
+from app.metrics import AUTH_BOOTSTRAP, HTTP_DURATION, HTTP_REQUESTS, RATE_LIMIT_DECISIONS
 from app.notification_routes import router as notification_router
 from app.notification_service import synchronize_source_tokens
 from app.realtime import RealtimeFanout
@@ -86,6 +87,7 @@ from app.services import (
     start_session,
 )
 from app.settings import get_settings
+from app.telemetry import add_trace_context, current_trace_id, init_telemetry, instrument_fastapi
 
 
 def configure_logging(level: str) -> None:
@@ -93,6 +95,7 @@ def configure_logging(level: str) -> None:
     structlog.configure(
         processors=[
             structlog.contextvars.merge_contextvars,
+            add_trace_context,
             structlog.processors.TimeStamper(fmt="iso", utc=True, key="timestamp"),
             structlog.stdlib.add_log_level,
             structlog.processors.JSONRenderer(),
@@ -113,6 +116,7 @@ async def refresh_settings_cache(app: FastAPI) -> None:
 @asynccontextmanager
 async def lifespan(app: FastAPI):
     settings = get_settings()
+    telemetry = init_telemetry()
     configure_logging(settings.log_level)
     app.state.settings = settings
     app.state.db = Database(settings.database_url)
@@ -138,14 +142,18 @@ async def lifespan(app: FastAPI):
         await app.state.jwks.refresh()
     except Exception:
         structlog.get_logger().warning("jwks.warmup_failed")
-    yield
-    settings_task.cancel()
-    with suppress(asyncio.CancelledError):
-        await settings_task
-    await app.state.http.aclose()
-    await app.state.redis.aclose()
-    await app.state.redis_rt.aclose()
-    await app.state.db.close()
+    try:
+        yield
+    finally:
+        settings_task.cancel()
+        with suppress(asyncio.CancelledError):
+            await settings_task
+        await app.state.http.aclose()
+        await app.state.redis.aclose()
+        await app.state.redis_rt.aclose()
+        await app.state.db.close()
+        if telemetry:
+            telemetry.shutdown()
 
 
 app = FastAPI(
@@ -221,7 +229,7 @@ async def request_context(request: Request, call_next: Any) -> Response:
     except ValueError:
         request_id = str(uuid.uuid4())
     request.state.request_id = request_id
-    request.state.trace_id = request_trace_id(request)
+    request.state.trace_id = current_trace_id() or request_trace_id(request)
     request.state.user_agent_hash = user_agent_hash(request)
     request.state.started_at = time.monotonic()
     structlog.contextvars.clear_contextvars()
@@ -230,7 +238,6 @@ async def request_context(request: Request, call_next: Any) -> Response:
         trace_id=request.state.trace_id,
         ux_session_id=request.headers.get("X-Ux-Session-Id"),
         method=request.method,
-        route=request.url.path,
         **{"service.name": "api-backend"},
     )
     origin = request.headers.get("Origin")
@@ -252,10 +259,21 @@ async def request_context(request: Request, call_next: Any) -> Response:
     response.headers["X-Request-ID"] = request_id
     response.headers["X-Content-Type-Options"] = "nosniff"
     response.headers["Cache-Control"] = response.headers.get("Cache-Control", "no-store")
+    duration_seconds = time.monotonic() - request.state.started_at
+    route = getattr(request.scope.get("route"), "path", "unmatched")
+    metric_attributes = {
+        "service.name": "api-backend",
+        "http.route": route,
+        "http.request.method": request.method,
+        "http.response.status_class": f"{response.status_code // 100}xx",
+    }
+    HTTP_REQUESTS.add(1, metric_attributes)
+    HTTP_DURATION.record(duration_seconds, metric_attributes)
     log.info(
         "request.complete",
+        route=route,
         status_code=response.status_code,
-        duration_ms=round((time.monotonic() - request.state.started_at) * 1000, 2),
+        duration_ms=round(duration_seconds * 1000, 2),
     )
     return response
 
@@ -396,17 +414,21 @@ async def enforce_limit(
         retry_after = await request.app.state.rate_limiter.consume(key, limit, window)
     except DependencyFailure:
         if fail_closed:
+            RATE_LIMIT_DECISIONS.add(1, {"scope": identity_type, "outcome": "dependency_error"})
             raise DomainError(
                 "dependency_unavailable", 503, "Rate limit service is unavailable"
             ) from None
+        RATE_LIMIT_DECISIONS.add(1, {"scope": identity_type, "outcome": "bypass"})
         return
     if retry_after:
+        RATE_LIMIT_DECISIONS.add(1, {"scope": identity_type, "outcome": "denied"})
         raise DomainError(
             "rate_limit_exceeded",
             429,
             "Rate limit exceeded",
             {"retry_after": retry_after},
         )
+    RATE_LIMIT_DECISIONS.add(1, {"scope": identity_type, "outcome": "allowed"})
 
 
 @app.get("/health/live", tags=["health"])
@@ -561,7 +583,13 @@ async def content(
 async def auth_bootstrap(
     body: BootstrapRequest, request: Request, db: Session, auth: PrincipalDep, settings: SnapshotDep
 ):
-    return await bootstrap(db, auth, body, settings, audit_context(request))
+    try:
+        result = await bootstrap(db, auth, body, settings, audit_context(request))
+    except Exception:
+        AUTH_BOOTSTRAP.add(1, {"outcome": "error"})
+        raise
+    AUTH_BOOTSTRAP.add(1, {"outcome": "success"})
+    return result
 
 
 @app.post("/api/v1/consents", status_code=201, tags=["auth"])
@@ -1106,6 +1134,9 @@ async def realtime(websocket: WebSocket):
         return
 
 
+instrument_fastapi(app)
+
+
 def run() -> None:
     settings = get_settings()
     uvicorn.run(