Admin/han-app
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Добавлен OTLP-провайдер, реализовано отбрасывание метрик и трейсов в observability + добавлен перезапуск nginx при пересборке контейнеров (ошибка, когда докер меняет адреса сервисов)

Browse Source
This commit is contained in:
mi
2026-07-29 15:15:40 +03:00
parent 3ed7239efa
commit 41e19005fb
43 changed files with 3016 additions and 83 deletions
Download Patch File Download Diff File Expand all files Collapse all files
codebase/backend/api-backend/app/main.py
+43 -12
View File
@@ -50,6 +50,7 @@ from app.integrations import (
S3Client,
SafetyClient,
)
from app.metrics import AUTH_BOOTSTRAP, HTTP_DURATION, HTTP_REQUESTS, RATE_LIMIT_DECISIONS
from app.notification_routes import router as notification_router
from app.notification_service import synchronize_source_tokens
from app.realtime import RealtimeFanout
@@ -86,6 +87,7 @@ from app.services import (
start_session,
)
from app.settings import get_settings
from app.telemetry import add_trace_context, current_trace_id, init_telemetry, instrument_fastapi
def configure_logging(level: str) -> None:
@@ -93,6 +95,7 @@ def configure_logging(level: str) -> None:
structlog.configure(
processors=[
structlog.contextvars.merge_contextvars,
add_trace_context,
structlog.processors.TimeStamper(fmt="iso", utc=True, key="timestamp"),
structlog.stdlib.add_log_level,
structlog.processors.JSONRenderer(),
@@ -113,6 +116,7 @@ async def refresh_settings_cache(app: FastAPI) -> None:
@asynccontextmanager
async def lifespan(app: FastAPI):
settings = get_settings()
telemetry = init_telemetry()
configure_logging(settings.log_level)
app.state.settings = settings
app.state.db = Database(settings.database_url)
@@ -138,14 +142,18 @@ async def lifespan(app: FastAPI):
await app.state.jwks.refresh()
except Exception:
structlog.get_logger().warning("jwks.warmup_failed")
yield
settings_task.cancel()
with suppress(asyncio.CancelledError):
await settings_task
await app.state.http.aclose()
await app.state.redis.aclose()
await app.state.redis_rt.aclose()
await app.state.db.close()
try:
yield
finally:
settings_task.cancel()
with suppress(asyncio.CancelledError):
await settings_task
await app.state.http.aclose()
await app.state.redis.aclose()
await app.state.redis_rt.aclose()
await app.state.db.close()
if telemetry:
telemetry.shutdown()
app = FastAPI(
@@ -221,7 +229,7 @@ async def request_context(request: Request, call_next: Any) -> Response:
except ValueError:
request_id = str(uuid.uuid4())
request.state.request_id = request_id
request.state.trace_id = request_trace_id(request)
request.state.trace_id = current_trace_id() or request_trace_id(request)
request.state.user_agent_hash = user_agent_hash(request)
request.state.started_at = time.monotonic()
structlog.contextvars.clear_contextvars()
@@ -230,7 +238,6 @@ async def request_context(request: Request, call_next: Any) -> Response:
trace_id=request.state.trace_id,
ux_session_id=request.headers.get("X-Ux-Session-Id"),
method=request.method,
route=request.url.path,
**{"service.name": "api-backend"},
)
origin = request.headers.get("Origin")
@@ -252,10 +259,21 @@ async def request_context(request: Request, call_next: Any) -> Response:
response.headers["X-Request-ID"] = request_id
response.headers["X-Content-Type-Options"] = "nosniff"
response.headers["Cache-Control"] = response.headers.get("Cache-Control", "no-store")
duration_seconds = time.monotonic() - request.state.started_at
route = getattr(request.scope.get("route"), "path", "unmatched")
metric_attributes = {
"service.name": "api-backend",
"http.route": route,
"http.request.method": request.method,
"http.response.status_class": f"{response.status_code // 100}xx",
}
HTTP_REQUESTS.add(1, metric_attributes)
HTTP_DURATION.record(duration_seconds, metric_attributes)
log.info(
"request.complete",
route=route,
status_code=response.status_code,
duration_ms=round((time.monotonic() - request.state.started_at) * 1000, 2),
duration_ms=round(duration_seconds * 1000, 2),
)
return response
@@ -396,17 +414,21 @@ async def enforce_limit(
retry_after = await request.app.state.rate_limiter.consume(key, limit, window)
except DependencyFailure:
if fail_closed:
RATE_LIMIT_DECISIONS.add(1, {"scope": identity_type, "outcome": "dependency_error"})
raise DomainError(
"dependency_unavailable", 503, "Rate limit service is unavailable"
) from None
RATE_LIMIT_DECISIONS.add(1, {"scope": identity_type, "outcome": "bypass"})
return
if retry_after:
RATE_LIMIT_DECISIONS.add(1, {"scope": identity_type, "outcome": "denied"})
raise DomainError(
"rate_limit_exceeded",
429,
"Rate limit exceeded",
{"retry_after": retry_after},
)
RATE_LIMIT_DECISIONS.add(1, {"scope": identity_type, "outcome": "allowed"})
@app.get("/health/live", tags=["health"])
@@ -561,7 +583,13 @@ async def content(
async def auth_bootstrap(
body: BootstrapRequest, request: Request, db: Session, auth: PrincipalDep, settings: SnapshotDep
):
return await bootstrap(db, auth, body, settings, audit_context(request))
try:
result = await bootstrap(db, auth, body, settings, audit_context(request))
except Exception:
AUTH_BOOTSTRAP.add(1, {"outcome": "error"})
raise
AUTH_BOOTSTRAP.add(1, {"outcome": "success"})
return result
@app.post("/api/v1/consents", status_code=201, tags=["auth"])
@@ -1106,6 +1134,9 @@ async def realtime(websocket: WebSocket):
return
instrument_fastapi(app)
def run() -> None:
settings = get_settings()
uvicorn.run(
codebase/backend/api-backend/app/metrics.py
+21
View File
@@ -0,0 +1,21 @@
from opentelemetry import metrics
meter = metrics.get_meter("han.api")
HTTP_REQUESTS = meter.create_counter(
"han_http_requests_total",
description="Completed HTTP requests",
)
HTTP_DURATION = meter.create_histogram(
"han_http_request_duration_seconds",
unit="s",
description="HTTP request duration",
)
AUTH_BOOTSTRAP = meter.create_counter(
"han_auth_bootstrap_total",
description="Authentication bootstrap outcomes",
)
RATE_LIMIT_DECISIONS = meter.create_counter(
"han_rate_limit_decisions_total",
description="Rate-limit decisions",
)
codebase/backend/api-backend/app/telemetry.py
+111
View File
@@ -0,0 +1,111 @@
from __future__ import annotations
import os
from dataclasses import dataclass
from typing import Any
from fastapi import FastAPI
from opentelemetry import metrics, trace
from opentelemetry.exporter.otlp.proto.grpc.metric_exporter import OTLPMetricExporter
from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter
from opentelemetry.instrumentation.botocore import BotocoreInstrumentor
from opentelemetry.instrumentation.fastapi import FastAPIInstrumentor
from opentelemetry.instrumentation.httpx import HTTPXClientInstrumentor
from opentelemetry.instrumentation.redis import RedisInstrumentor
from opentelemetry.instrumentation.sqlalchemy import SQLAlchemyInstrumentor
from opentelemetry.propagate import set_global_textmap
from opentelemetry.sdk.metrics import MeterProvider
from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader
from opentelemetry.sdk.resources import Resource
from opentelemetry.sdk.trace import TracerProvider
from opentelemetry.sdk.trace.export import BatchSpanProcessor
from opentelemetry.sdk.trace.sampling import ALWAYS_ON
from opentelemetry.trace.propagation.tracecontext import TraceContextTextMapPropagator
@dataclass(slots=True)
class TelemetryRuntime:
tracer_provider: TracerProvider
meter_provider: MeterProvider
def shutdown(self) -> None:
self.meter_provider.shutdown()
self.tracer_provider.shutdown()
_runtime: TelemetryRuntime | None = None
def _resource(service_name: str) -> Resource:
return Resource.create(
{
"service.name": service_name,
"service.namespace": "han-chat",
"service.version": os.getenv("RELEASE_VERSION", "unknown"),
"deployment.environment": os.getenv("APP_ENV", "production-like"),
}
)
def init_telemetry(service_name: str | None = None) -> TelemetryRuntime | None:
global _runtime
if _runtime is not None:
return _runtime
endpoint = os.getenv("OTEL_EXPORTER_OTLP_ENDPOINT", "").strip()
if not endpoint:
return None
resource = _resource(service_name or os.getenv("OTEL_SERVICE_NAME", "api-backend"))
insecure = endpoint.startswith("http://")
set_global_textmap(TraceContextTextMapPropagator())
tracer_provider = TracerProvider(resource=resource, sampler=ALWAYS_ON)
tracer_provider.add_span_processor(
BatchSpanProcessor(
OTLPSpanExporter(endpoint=endpoint, insecure=insecure, timeout=3),
max_queue_size=2048,
schedule_delay_millis=5000,
max_export_batch_size=512,
export_timeout_millis=3000,
)
)
trace.set_tracer_provider(tracer_provider)
metric_reader = PeriodicExportingMetricReader(
OTLPMetricExporter(endpoint=endpoint, insecure=insecure, timeout=3),
export_interval_millis=30000,
export_timeout_millis=3000,
)
meter_provider = MeterProvider(resource=resource, metric_readers=[metric_reader])
metrics.set_meter_provider(meter_provider)
HTTPXClientInstrumentor().instrument()
SQLAlchemyInstrumentor().instrument(enable_commenter=False)
RedisInstrumentor().instrument()
BotocoreInstrumentor().instrument()
_runtime = TelemetryRuntime(tracer_provider, meter_provider)
return _runtime
def instrument_fastapi(app: FastAPI) -> None:
FastAPIInstrumentor.instrument_app(
app,
excluded_urls="/health/live,/health/ready,/nginx-health/live",
)
def add_trace_context(
_logger: Any,
_method_name: str,
event_dict: dict[str, Any],
) -> dict[str, Any]:
context = trace.get_current_span().get_span_context()
if context.is_valid:
event_dict["trace_id"] = format(context.trace_id, "032x")
event_dict["span_id"] = format(context.span_id, "016x")
return event_dict
def current_trace_id() -> str | None:
context = trace.get_current_span().get_span_context()
return format(context.trace_id, "032x") if context.is_valid else None