ВМ1: реализован сбор логов

2026-09-07 12:46:45 +03:00
parent 44db38f6fe
commit 85df788f2d
42 changed files with 1669 additions and 310 deletions
@@ -0,0 +1,63 @@
+"""Persist request and W3C trace context for inbox forwarding.
+
+Revision ID: 0002_inbox_trace_context
+Revises: 0001_bitrix_local_schema
+Create Date: 2026-09-03
+"""
+
+from collections.abc import Sequence
+
+from alembic import op
+
+revision: str = "0002_inbox_trace_context"
+down_revision: str | None = "0001_bitrix_local"
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
+
+
+def upgrade() -> None:
+    op.execute(
+        """
+        ALTER TABLE bitrix_local.inbox_events
+          ADD COLUMN IF NOT EXISTS request_id varchar(64),
+          ADD COLUMN IF NOT EXISTS traceparent varchar(55)
+        """
+    )
+    op.execute(
+        """
+        ALTER TABLE bitrix_local.outbound_messages
+          ADD COLUMN IF NOT EXISTS request_id varchar(64),
+          ADD COLUMN IF NOT EXISTS traceparent varchar(55)
+        """
+    )
+    op.execute(
+        """
+        ALTER TABLE bitrix_local.delivery_ack_outbox
+          ADD COLUMN IF NOT EXISTS request_id varchar(64),
+          ADD COLUMN IF NOT EXISTS traceparent varchar(55)
+        """
+    )
+
+
+def downgrade() -> None:
+    op.execute(
+        """
+        ALTER TABLE bitrix_local.inbox_events
+          DROP COLUMN IF EXISTS traceparent,
+          DROP COLUMN IF EXISTS request_id
+        """
+    )
+    op.execute(
+        """
+        ALTER TABLE bitrix_local.outbound_messages
+          DROP COLUMN IF EXISTS traceparent,
+          DROP COLUMN IF EXISTS request_id
+        """
+    )
+    op.execute(
+        """
+        ALTER TABLE bitrix_local.delivery_ack_outbox
+          DROP COLUMN IF EXISTS traceparent,
+          DROP COLUMN IF EXISTS request_id
+        """
+    )
@@ -0,0 +1,47 @@
+from __future__ import annotations
+
+import re
+from collections.abc import Mapping, MutableMapping
+from typing import Any
+
+REDACTED = "[REDACTED]"
+_SENSITIVE_KEY = re.compile(
+    r"(authorization|cookie|password|passwd|secret|token|api[_-]?key|"
+    r"database[_-]?url|dsn|callback[_-]?url|phone|email|message|payload)",
+    re.IGNORECASE,
+)
+_URI_USERINFO = re.compile(r"(?P<scheme>[a-z][a-z0-9+.-]*://)[^/@\s]+@", re.IGNORECASE)
+_QUERY_SECRET = re.compile(
+    r"(?P<prefix>[?&](?:token|access_token|api_key|key|secret|password)=)[^&#\s]+",
+    re.IGNORECASE,
+)
+_AUTH_VALUE = re.compile(r"\b(Bearer|Basic)\s+[A-Za-z0-9._~+/=-]+", re.IGNORECASE)
+
+
+def sanitize_text(value: str) -> str:
+    value = _URI_USERINFO.sub(r"\g<scheme>[REDACTED]@", value)
+    value = _QUERY_SECRET.sub(r"\g<prefix>[REDACTED]", value)
+    return _AUTH_VALUE.sub(r"\1 [REDACTED]", value)
+
+
+def sanitize_value(value: Any) -> Any:
+    if isinstance(value, str):
+        return sanitize_text(value)
+    if isinstance(value, Mapping):
+        return {
+            str(key): REDACTED if _SENSITIVE_KEY.search(str(key)) else sanitize_value(item)
+            for key, item in value.items()
+        }
+    if isinstance(value, list):
+        return [sanitize_value(item) for item in value]
+    if isinstance(value, tuple):
+        return tuple(sanitize_value(item) for item in value)
+    return value
+
+
+def redact_event(
+    _logger: Any,
+    _method_name: str,
+    event_dict: MutableMapping[str, Any],
+) -> MutableMapping[str, Any]:
+    return sanitize_value(event_dict)
@@ -16,11 +16,13 @@ from typing import Annotated, Any, Literal
 from urllib.parse import quote, urlparse

 import httpx
+import structlog
 import uvicorn
 from cryptography.hazmat.primitives.ciphers.aead import AESGCM
 from fastapi import Depends, FastAPI, Header, HTTPException, Request
 from fastapi.exceptions import RequestValidationError
 from fastapi.responses import HTMLResponse, JSONResponse
+from opentelemetry import trace
 from pydantic import BaseModel, ConfigDict, Field, model_validator
 from pydantic_settings import BaseSettings, SettingsConfigDict
 from sqlalchemy import func, or_, select, text
@@ -37,9 +39,19 @@ from app.models import (
    PortalInstallation,
    now,
 )
+from app.logging_security import redact_event
 from app.postgres import create_postgres_engine
+from app.telemetry import (
+    TelemetryRuntime,
+    add_trace_context,
+    current_traceparent,
+    init_telemetry,
+    instrument_fastapi,
+    origin_links,
+)

-logger = logging.getLogger("bitrix-local-app")
+logger = structlog.get_logger("bitrix-local-app")
+EXPECTED_BITRIX_DB_REVISION = "0002_inbox_trace_context"

 BITRIX_SENDER_PREFIX = re.compile(
    r"^\[b\][^\r\n\[]+:\[/b\]\s*(?:\[br\]\s*)?",
@@ -57,6 +69,7 @@ CONNECTOR_ICON_DATA_URI = "data:image/svg+xml," + quote(
 class Settings(BaseSettings):
    model_config = SettingsConfigDict(extra="ignore")
    app_env: str = "production-like"
+    log_level: str = "INFO"
    bitrix_database_url: str
    bitrix_client_id: str
    bitrix_client_secret: str
@@ -78,6 +91,28 @@ class Settings(BaseSettings):
    bitrix_worker_poll_sec: float = Field(default=1, ge=0.05, le=30)


+def configure_logging(level: str, telemetry: TelemetryRuntime | None = None) -> None:
+    logging.basicConfig(level=level, format="%(message)s")
+    if telemetry and telemetry.logging_handler not in logging.getLogger().handlers:
+        telemetry.logging_handler.addFilter(
+            lambda record: not record.name.startswith("opentelemetry")
+        )
+        logging.getLogger().addHandler(telemetry.logging_handler)
+    structlog.configure(
+        processors=[
+            structlog.contextvars.merge_contextvars,
+            add_trace_context,
+            redact_event,
+            structlog.processors.TimeStamper(fmt="iso", utc=True, key="timestamp"),
+            structlog.stdlib.add_log_level,
+            structlog.processors.JSONRenderer(),
+        ],
+        logger_factory=structlog.stdlib.LoggerFactory(),
+        wrapper_class=structlog.stdlib.BoundLogger,
+        cache_logger_on_first_use=True,
+    )
+
+
 class TokenCipher:
    def __init__(self, encoded_key: str, version: str) -> None:
        try:
@@ -309,8 +344,8 @@ def normalize_event(payload: dict[str, Any]) -> dict[str, Any] | None:
                normalized_file["_bitrix_file_id"] = str(file_id)
            if not normalized_file["download_url"] and not file_id:
                logger.warning(
-                    "inbound file has no supported download reference; keys=%s",
-                    sorted(str(key) for key in item),
+                    "inbound_file.unsupported_reference",
+                    keys=sorted(str(key) for key in item),
                )
            files.append(normalized_file)
        if not text_value and not files:
@@ -498,6 +533,8 @@ def create_app(settings: Settings | None = None) -> FastAPI:

    @asynccontextmanager
    async def lifespan(app: FastAPI):
+        telemetry = init_telemetry()
+        configure_logging(cfg.log_level, telemetry)
        engine = create_postgres_engine(
            cfg.bitrix_database_url,
            pool_size=5,
@@ -527,6 +564,8 @@ def create_app(settings: Settings | None = None) -> FastAPI:
                await task
        await app.state.http.aclose()
        await engine.dispose()
+        if telemetry:
+            telemetry.shutdown()

    app = FastAPI(
        title="HAN Bitrix24 Local App",
@@ -540,9 +579,13 @@ def create_app(settings: Settings | None = None) -> FastAPI:
    @app.middleware("http")
    async def request_context(request: Request, call_next):
        request.state.request_id = request.headers.get("X-Request-ID") or str(uuid.uuid4())
-        response = await call_next(request)
-        response.headers["X-Request-ID"] = request.state.request_id
-        return response
+        with structlog.contextvars.bound_contextvars(
+            request_id=request.state.request_id,
+            **{"service.name": "bitrix-local-app"},
+        ):
+            response = await call_next(request)
+            response.headers["X-Request-ID"] = request.state.request_id
+            return response

    @app.exception_handler(HTTPException)
    async def http_error(_: Request, exc: HTTPException):
@@ -579,9 +622,22 @@ def create_app(settings: Settings | None = None) -> FastAPI:
            async with request.app.state.sessions() as session:
                portal = await active_portal(session)
                await session.scalar(select(func.now()))
+                revision = await session.scalar(
+                    text("SELECT version_num FROM bitrix_local.alembic_version")
+                )
            worker_ok = all(not task.done() for task in request.app.state.workers)
-            if portal and portal.install_status == "installed" and worker_ok:
+            if (
+                portal
+                and portal.install_status == "installed"
+                and worker_ok
+                and revision == EXPECTED_BITRIX_DB_REVISION
+            ):
                return {"status": "ready", "portal": "installed", "workers": "running"}
+            if revision != EXPECTED_BITRIX_DB_REVISION:
+                return JSONResponse(
+                    {"status": "not_ready", "reason": "migration_required"},
+                    status_code=503,
+                )
            return JSONResponse(
                {"status": "not_ready", "reason": "portal_not_installed"}, status_code=503
            )
@@ -655,7 +711,12 @@ def create_app(settings: Settings | None = None) -> FastAPI:
            normalized = normalize_event(payload)
            if normalized is None:
                return {"status": "ignored"}
-            created = await save_inbox(request.app, normalized)
+            created = await save_inbox(
+                request.app,
+                normalized,
+                request.state.request_id,
+                current_traceparent(),
+            )
            return JSONResponse(
                {"status": "accepted" if created else "duplicate"},
                status_code=202 if created else 200,
@@ -731,6 +792,8 @@ def create_app(settings: Settings | None = None) -> FastAPI:
                    external_chat_id=dto.external_chat_id,
                    request_fingerprint=fp,
                    payload_json=body,
+                    request_id=request.state.request_id,
+                    traceparent=current_traceparent(),
                    status="sending",
                    lease_until=now() + timedelta(seconds=cfg.bitrix_http_timeout_sec + 5),
                )
@@ -768,12 +831,10 @@ def create_app(settings: Settings | None = None) -> FastAPI:
            )
        except Exception as exc:
            logger.error(
-                "outbound delivery failed",
-                extra={
-                    "request_id": request.state.request_id,
-                    "message_id": str(dto.message_id),
-                    "error_type": type(exc).__name__,
-                },
+                "outbound.delivery_failed",
+                request_id=request.state.request_id,
+                message_id=str(dto.message_id),
+                error_type=type(exc).__name__,
            )
            async with request.app.state.sessions() as session:
                current = await session.get(OutboundMessage, row.id, with_for_update=True)
@@ -845,6 +906,7 @@ def create_app(settings: Settings | None = None) -> FastAPI:
        result = await reconcile_setup(request.app)
        return {"status": "completed" if all(result.values()) else "partial", "steps": result}

+    instrument_fastapi(app)
    return app


@@ -960,7 +1022,12 @@ async def uninstall_payload(app: FastAPI, payload: dict[str, Any]) -> None:
            await session.commit()


-async def save_inbox(app: FastAPI, normalized: dict[str, Any]) -> bool:
+async def save_inbox(
+    app: FastAPI,
+    normalized: dict[str, Any],
+    request_id: str,
+    traceparent: str | None,
+) -> bool:
    fingerprint = hashlib.sha256(
        json.dumps(normalized, sort_keys=True, separators=(",", ":")).encode()
    ).hexdigest()
@@ -971,6 +1038,8 @@ async def save_inbox(app: FastAPI, normalized: dict[str, Any]) -> bool:
        bitrix_message_id=normalized["bitrix_message_id"],
        payload_fingerprint=fingerprint,
        normalized_json=normalized,
+        request_id=request_id,
+        traceparent=traceparent,
    )
    async with app.state.sessions() as session:
        session.add(row)
@@ -1168,8 +1237,9 @@ async def worker_loop(app: FastAPI, kind: str) -> None:
                await process_setup(app)
        except Exception as exc:
            logger.error(
-                "worker iteration failed",
-                extra={"worker_kind": kind, "error_type": type(exc).__name__},
+                "worker.iteration_failed",
+                worker_kind=kind,
+                error_type=type(exc).__name__,
            )
        try:
            await asyncio.wait_for(app.state.stop.wait(), app.state.settings.bitrix_worker_poll_sec)
@@ -1182,45 +1252,57 @@ async def process_inbox(app: FastAPI) -> None:
        row = await claim_one(session, InboxEvent, ["received", "retry"])
    if not row:
        return
-    try:
-        payload = json.loads(json.dumps(row.normalized_json))
-        await resolve_inbound_file_urls(app, payload)
-        response = await app.state.http.post(
-            app.state.settings.bitrix_api_forward_url,
-            json=payload,
-            headers={
-                "Authorization": f"Bearer {app.state.settings.bitrix_api_forward_token}",
-                "X-Request-ID": str(uuid.uuid4()),
-            },
-            follow_redirects=False,
-        )
-        duplicate = response.status_code in {200, 204}
-        if response.status_code != 201 and not duplicate:
-            response.raise_for_status()
-        async with app.state.sessions() as session:
-            current = await session.get(InboxEvent, row.id, with_for_update=True)
-            current.status = "ack_pending"
-            current.api_ack_status = "duplicate" if duplicate else "created"
-            current.lease_until = None
-            session.add(
-                DeliveryAckOutbox(
-                    inbox_event_id=current.id,
-                    payload_json={
-                        "external_chat_id": str(current.external_chat_id),
-                        "bitrix_message_id": current.bitrix_message_id,
-                    },
-                )
+    tracer = trace.get_tracer("han.bitrix.inbox")
+    with (
+        tracer.start_as_current_span(
+            "bitrix.inbox.forward",
+            links=origin_links(row.traceparent),
+        ),
+        structlog.contextvars.bound_contextvars(
+            request_id=row.request_id,
+            inbox_id=str(row.id),
+            **{"service.name": "bitrix-local-app"},
+        ),
+    ):
+        try:
+            payload = json.loads(json.dumps(row.normalized_json))
+            await resolve_inbound_file_urls(app, payload)
+            response = await app.state.http.post(
+                app.state.settings.bitrix_api_forward_url,
+                json=payload,
+                headers={
+                    "Authorization": f"Bearer {app.state.settings.bitrix_api_forward_token}",
+                    "X-Request-ID": row.request_id or str(uuid.uuid4()),
+                },
+                follow_redirects=False,
            )
-            await session.commit()
-    except Exception as exc:
-        logger.error(
-            "inbound forward failed",
-            extra={
-                "inbox_id": str(row.id),
-                "error_type": type(exc).__name__,
-            },
-        )
-        await mark_retry(app, InboxEvent, row.id, "api_forward_failed")
+            duplicate = response.status_code in {200, 204}
+            if response.status_code != 201 and not duplicate:
+                response.raise_for_status()
+            async with app.state.sessions() as session:
+                current = await session.get(InboxEvent, row.id, with_for_update=True)
+                current.status = "ack_pending"
+                current.api_ack_status = "duplicate" if duplicate else "created"
+                current.lease_until = None
+                session.add(
+                    DeliveryAckOutbox(
+                        inbox_event_id=current.id,
+                        request_id=current.request_id,
+                        traceparent=current.traceparent,
+                        payload_json={
+                            "external_chat_id": str(current.external_chat_id),
+                            "bitrix_message_id": current.bitrix_message_id,
+                        },
+                    )
+                )
+                await session.commit()
+        except Exception as exc:
+            logger.error(
+                "inbound.forward_failed",
+                inbox_id=str(row.id),
+                error_type=type(exc).__name__,
+            )
+            await mark_retry(app, InboxEvent, row.id, "api_forward_failed")


 async def resolve_inbound_file_urls(app: FastAPI, payload: dict[str, Any]) -> None:
@@ -1252,27 +1334,41 @@ async def process_ack(app: FastAPI) -> None:
        portal = await active_portal(session)
    if not row or not portal:
        return
-    try:
-        await app.state.bitrix.call(
-            portal,
-            "imconnector.send.status.delivery",
-            {
-                "CONNECTOR": app.state.settings.bitrix_connector_id,
-                "LINE": app.state.settings.bitrix_open_line_id,
-                "MESSAGES[0][im][chat_id]": row.payload_json["external_chat_id"],
-                "MESSAGES[0][message][id]": row.payload_json["bitrix_message_id"] or "",
-            },
-        )
-        async with app.state.sessions() as session:
-            current = await session.get(DeliveryAckOutbox, row.id, with_for_update=True)
-            event = await session.get(InboxEvent, current.inbox_event_id, with_for_update=True)
-            current.status = "completed"
-            current.lease_until = None
-            event.status = "completed"
-            event.delivery_ack_status = "sent"
-            await session.commit()
-    except Exception:
-        await mark_retry(app, DeliveryAckOutbox, row.id, "delivery_ack_failed")
+    tracer = trace.get_tracer("han.bitrix.ack")
+    with (
+        tracer.start_as_current_span(
+            "bitrix.delivery_ack.process",
+            links=origin_links(row.traceparent),
+        ),
+        structlog.contextvars.bound_contextvars(
+            request_id=row.request_id,
+            ack_outbox_id=str(row.id),
+            **{"service.name": "bitrix-local-app"},
+        ),
+    ):
+        try:
+            await app.state.bitrix.call(
+                portal,
+                "imconnector.send.status.delivery",
+                {
+                    "CONNECTOR": app.state.settings.bitrix_connector_id,
+                    "LINE": app.state.settings.bitrix_open_line_id,
+                    "MESSAGES[0][im][chat_id]": row.payload_json["external_chat_id"],
+                    "MESSAGES[0][message][id]": row.payload_json["bitrix_message_id"] or "",
+                },
+            )
+            async with app.state.sessions() as session:
+                current = await session.get(DeliveryAckOutbox, row.id, with_for_update=True)
+                event = await session.get(
+                    InboxEvent, current.inbox_event_id, with_for_update=True
+                )
+                current.status = "completed"
+                current.lease_until = None
+                event.status = "completed"
+                event.delivery_ack_status = "sent"
+                await session.commit()
+        except Exception:
+            await mark_retry(app, DeliveryAckOutbox, row.id, "delivery_ack_failed")


 async def process_outbound(app: FastAPI) -> None:
@@ -1285,20 +1381,30 @@ async def process_outbound(app: FastAPI) -> None:
        )
    if not row:
        return
-    try:
-        await asyncio.wait_for(
-            deliver_outbound(app, row.id),
-            timeout=app.state.settings.bitrix_http_timeout_sec,
-        )
-    except Exception as exc:
-        logger.error(
-            "outbound retry failed",
-            extra={
-                "outbound_id": str(row.id),
-                "error_type": type(exc).__name__,
-            },
-        )
-        await mark_retry(app, OutboundMessage, row.id, "bitrix_delivery_failed")
+    tracer = trace.get_tracer("han.bitrix.outbound")
+    with (
+        tracer.start_as_current_span(
+            "bitrix.outbound.process",
+            links=origin_links(row.traceparent),
+        ),
+        structlog.contextvars.bound_contextvars(
+            request_id=row.request_id,
+            outbound_id=str(row.id),
+            **{"service.name": "bitrix-local-app"},
+        ),
+    ):
+        try:
+            await asyncio.wait_for(
+                deliver_outbound(app, row.id),
+                timeout=app.state.settings.bitrix_http_timeout_sec,
+            )
+        except Exception as exc:
+            logger.error(
+                "outbound.retry_failed",
+                outbound_id=str(row.id),
+                error_type=type(exc).__name__,
+            )
+            await mark_retry(app, OutboundMessage, row.id, "bitrix_delivery_failed")


 async def process_setup(app: FastAPI) -> None:
@@ -133,6 +133,8 @@ class InboxEvent(Common, Base):
    bitrix_message_id: Mapped[str | None] = mapped_column(String(255))
    payload_fingerprint: Mapped[str] = mapped_column(String(64))
    normalized_json: Mapped[dict] = mapped_column(JSON)
+    request_id: Mapped[str | None] = mapped_column(String(64))
+    traceparent: Mapped[str | None] = mapped_column(String(55))
    status: Mapped[str] = mapped_column(String(32), default="received")
    attempt_count: Mapped[int] = mapped_column(Integer, default=0)
    next_attempt_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=now)
@@ -149,6 +151,8 @@ class OutboundMessage(Common, Base):
    external_chat_id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True))
    request_fingerprint: Mapped[str] = mapped_column(String(64))
    payload_json: Mapped[dict] = mapped_column(JSON)
+    request_id: Mapped[str | None] = mapped_column(String(64))
+    traceparent: Mapped[str | None] = mapped_column(String(55))
    status: Mapped[str] = mapped_column(String(32), default="received")
    bitrix_message_id: Mapped[str | None] = mapped_column(String(255))
    response_json: Mapped[dict | None] = mapped_column(JSON)
@@ -165,6 +169,8 @@ class DeliveryAckOutbox(Common, Base):
        UUID(as_uuid=True), ForeignKey(f"{SCHEMA}.inbox_events.id"), unique=True
    )
    payload_json: Mapped[dict] = mapped_column(JSON)
+    request_id: Mapped[str | None] = mapped_column(String(64))
+    traceparent: Mapped[str | None] = mapped_column(String(55))
    status: Mapped[str] = mapped_column(String(32), default="pending")
    attempt_count: Mapped[int] = mapped_column(Integer, default=0)
    next_attempt_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=now)
@@ -0,0 +1,119 @@
+from __future__ import annotations
+
+import logging
+import os
+from collections.abc import MutableMapping
+from dataclasses import dataclass
+from typing import Any
+
+from fastapi import FastAPI
+from opentelemetry import metrics, trace
+from opentelemetry._logs import set_logger_provider
+from opentelemetry.exporter.otlp.proto.grpc._log_exporter import OTLPLogExporter
+from opentelemetry.exporter.otlp.proto.grpc.metric_exporter import OTLPMetricExporter
+from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter
+from opentelemetry.instrumentation.fastapi import FastAPIInstrumentor
+from opentelemetry.instrumentation.httpx import HTTPXClientInstrumentor
+from opentelemetry.instrumentation.sqlalchemy import SQLAlchemyInstrumentor
+from opentelemetry.propagate import extract, inject, set_global_textmap
+from opentelemetry.sdk._logs import LoggerProvider, LoggingHandler
+from opentelemetry.sdk._logs.export import BatchLogRecordProcessor
+from opentelemetry.sdk.metrics import MeterProvider
+from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader
+from opentelemetry.sdk.resources import Resource
+from opentelemetry.sdk.trace import TracerProvider
+from opentelemetry.sdk.trace.export import BatchSpanProcessor
+from opentelemetry.sdk.trace.sampling import ALWAYS_ON
+from opentelemetry.trace.propagation.tracecontext import TraceContextTextMapPropagator
+
+
+@dataclass(slots=True)
+class TelemetryRuntime:
+    tracer_provider: TracerProvider
+    meter_provider: MeterProvider
+    logger_provider: LoggerProvider
+    logging_handler: LoggingHandler
+
+    def shutdown(self) -> None:
+        self.logger_provider.shutdown()
+        self.meter_provider.shutdown()
+        self.tracer_provider.shutdown()
+
+
+_runtime: TelemetryRuntime | None = None
+
+
+def init_telemetry() -> TelemetryRuntime | None:
+    global _runtime
+    if _runtime is not None:
+        return _runtime
+    endpoint = os.getenv("OTEL_EXPORTER_OTLP_ENDPOINT", "").strip()
+    if not endpoint:
+        return None
+    resource = Resource.create(
+        {
+            "service.name": os.getenv("OTEL_SERVICE_NAME", "bitrix-local-app"),
+            "service.namespace": "han-chat",
+            "service.version": os.getenv("RELEASE_VERSION", "unknown"),
+            "deployment.environment": os.getenv("APP_ENV", "production-like"),
+        }
+    )
+    insecure = endpoint.startswith("http://")
+    set_global_textmap(TraceContextTextMapPropagator())
+    tracer_provider = TracerProvider(resource=resource, sampler=ALWAYS_ON)
+    tracer_provider.add_span_processor(
+        BatchSpanProcessor(OTLPSpanExporter(endpoint=endpoint, insecure=insecure, timeout=3))
+    )
+    trace.set_tracer_provider(tracer_provider)
+    metric_reader = PeriodicExportingMetricReader(
+        OTLPMetricExporter(endpoint=endpoint, insecure=insecure, timeout=3),
+        export_interval_millis=30000,
+        export_timeout_millis=3000,
+    )
+    meter_provider = MeterProvider(resource=resource, metric_readers=[metric_reader])
+    metrics.set_meter_provider(meter_provider)
+    logger_provider = LoggerProvider(resource=resource)
+    logger_provider.add_log_record_processor(
+        BatchLogRecordProcessor(
+            OTLPLogExporter(endpoint=endpoint, insecure=insecure, timeout=3),
+            max_queue_size=2048,
+            max_export_batch_size=512,
+            export_timeout_millis=3000,
+        )
+    )
+    set_logger_provider(logger_provider)
+    logging_handler = LoggingHandler(level=logging.NOTSET, logger_provider=logger_provider)
+    HTTPXClientInstrumentor().instrument()
+    SQLAlchemyInstrumentor().instrument(enable_commenter=False)
+    _runtime = TelemetryRuntime(
+        tracer_provider, meter_provider, logger_provider, logging_handler
+    )
+    return _runtime
+
+
+def instrument_fastapi(app: FastAPI) -> None:
+    FastAPIInstrumentor.instrument_app(app, excluded_urls="/health/live,/health/ready")
+
+
+def add_trace_context(
+    _logger: Any, _method_name: str, event_dict: MutableMapping[str, Any]
+) -> MutableMapping[str, Any]:
+    context = trace.get_current_span().get_span_context()
+    if context.is_valid:
+        event_dict["trace_id"] = format(context.trace_id, "032x")
+        event_dict["span_id"] = format(context.span_id, "016x")
+    return event_dict
+
+
+def current_traceparent() -> str | None:
+    carrier: dict[str, str] = {}
+    inject(carrier)
+    return carrier.get("traceparent")
+
+
+def origin_links(traceparent: str | None) -> list[trace.Link]:
+    if not traceparent:
+        return []
+    context = extract({"traceparent": traceparent})
+    span_context = trace.get_current_span(context).get_span_context()
+    return [trace.Link(span_context)] if span_context.is_valid else []
@@ -8,9 +8,16 @@ dependencies = [
  "cryptography>=45,<46",
  "fastapi>=0.116,<1",
  "httpx>=0.28,<1",
+  "opentelemetry-api>=1.44,<2",
+  "opentelemetry-exporter-otlp-proto-grpc>=1.44,<2",
+  "opentelemetry-instrumentation-fastapi>=0.65b0,<1",
+  "opentelemetry-instrumentation-httpx>=0.65b0,<1",
+  "opentelemetry-instrumentation-sqlalchemy>=0.65b0,<1",
+  "opentelemetry-sdk>=1.44,<2",
  "pydantic-settings>=2.10,<3",
  "python-multipart>=0.0.20,<1",
  "sqlalchemy[asyncio]>=2.0.41,<3",
+  "structlog>=25,<26",
  "uvicorn[standard]>=0.35,<1",
 ]

@@ -19,6 +19,7 @@ os.environ.setdefault(
 import httpx
 import pytest

+from app.logging_security import REDACTED, sanitize_value
 from app.main import (
    BitrixClient,
    TokenCipher,
@@ -30,6 +31,17 @@ from app.main import (
    safely_retryable,
    validate_portal,
 )
+from app.telemetry import add_trace_context, origin_links
+
+
+def test_bitrix_logs_redact_secrets_and_do_not_invent_trace_ids():
+    value = sanitize_value(
+        {"authorization": "Bearer secret", "url": "https://example.test/?token=secret"}
+    )
+    assert value["authorization"] == REDACTED
+    assert "secret" not in value["url"]
+    assert add_trace_context(None, "info", {"event": "safe"}) == {"event": "safe"}
+    assert origin_links("invalid") == []


 def test_token_cipher_binds_aad():