ВМ1: реализован сбор логов

2026-09-07 12:46:45 +03:00
parent 44db38f6fe
commit 85df788f2d
42 changed files with 1669 additions and 310 deletions
@@ -1,7 +1,7 @@
 from __future__ import annotations

 import re
-from collections.abc import Mapping
+from collections.abc import Mapping, MutableMapping
 from typing import Any

 REDACTED = "[REDACTED]"
@@ -42,6 +42,6 @@ def sanitize_value(value: Any) -> Any:
 def redact_event(
    _logger: Any,
    _method_name: str,
-    event_dict: dict[str, Any],
-) -> dict[str, Any]:
+    event_dict: MutableMapping[str, Any],
+) -> MutableMapping[str, Any]:
    return sanitize_value(event_dict)
@@ -32,13 +32,18 @@ from app.service import (
    read_message,
 )
 from app.settings import get_settings
-from app.telemetry import add_trace_context, init_telemetry, instrument_fastapi
+from app.telemetry import TelemetryRuntime, add_trace_context, init_telemetry, instrument_fastapi

 log = structlog.get_logger()


-def configure_logging(level: str) -> None:
+def configure_logging(level: str, telemetry: TelemetryRuntime | None = None) -> None:
    logging.basicConfig(level=level, format="%(message)s")
+    if telemetry and telemetry.logging_handler not in logging.getLogger().handlers:
+        telemetry.logging_handler.addFilter(
+            lambda record: not record.name.startswith("opentelemetry")
+        )
+        logging.getLogger().addHandler(telemetry.logging_handler)
    structlog.configure(
        processors=[
            structlog.contextvars.merge_contextvars,
@@ -47,7 +52,10 @@ def configure_logging(level: str) -> None:
            structlog.processors.TimeStamper(fmt="iso", utc=True, key="timestamp"),
            structlog.stdlib.add_log_level,
            structlog.processors.JSONRenderer(),
-        ]
+        ],
+        logger_factory=structlog.stdlib.LoggerFactory(),
+        wrapper_class=structlog.stdlib.BoundLogger,
+        cache_logger_on_first_use=True,
    )


@@ -55,7 +63,7 @@ def configure_logging(level: str) -> None:
 async def lifespan(app: FastAPI):
    settings = get_settings()
    telemetry = init_telemetry()
-    configure_logging(settings.log_level)
+    configure_logging(settings.log_level, telemetry)
    app.state.settings = settings
    app.state.db = Database(settings.database_url)
    try:
@@ -1,17 +1,23 @@
 from __future__ import annotations

+import logging
 import os
+from collections.abc import MutableMapping
 from dataclasses import dataclass
 from typing import Any

 from fastapi import FastAPI
 from opentelemetry import metrics, trace
+from opentelemetry._logs import set_logger_provider
+from opentelemetry.exporter.otlp.proto.grpc._log_exporter import OTLPLogExporter
 from opentelemetry.exporter.otlp.proto.grpc.metric_exporter import OTLPMetricExporter
 from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter
 from opentelemetry.instrumentation.fastapi import FastAPIInstrumentor
 from opentelemetry.instrumentation.httpx import HTTPXClientInstrumentor
 from opentelemetry.instrumentation.sqlalchemy import SQLAlchemyInstrumentor
 from opentelemetry.propagate import set_global_textmap
+from opentelemetry.sdk._logs import LoggerProvider, LoggingHandler
+from opentelemetry.sdk._logs.export import BatchLogRecordProcessor
 from opentelemetry.sdk.metrics import MeterProvider
 from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader
 from opentelemetry.sdk.resources import Resource
@@ -25,8 +31,11 @@ from opentelemetry.trace.propagation.tracecontext import TraceContextTextMapProp
 class TelemetryRuntime:
    tracer_provider: TracerProvider
    meter_provider: MeterProvider
+    logger_provider: LoggerProvider
+    logging_handler: LoggingHandler

    def shutdown(self) -> None:
+        self.logger_provider.shutdown()
        self.meter_provider.shutdown()
        self.tracer_provider.shutdown()

@@ -53,7 +62,9 @@ def init_telemetry(service_name: str | None = None) -> TelemetryRuntime | None:
    if not endpoint:
        return None

-    resource = _resource(service_name or os.getenv("OTEL_SERVICE_NAME", "sms-service"))
+    resource = _resource(
+        service_name or os.getenv("OTEL_SERVICE_NAME", "sms-service") or "sms-service"
+    )
    insecure = endpoint.startswith("http://")
    set_global_textmap(TraceContextTextMapPropagator())

@@ -77,9 +88,27 @@ def init_telemetry(service_name: str | None = None) -> TelemetryRuntime | None:
    meter_provider = MeterProvider(resource=resource, metric_readers=[metric_reader])
    metrics.set_meter_provider(meter_provider)

+    logger_provider = LoggerProvider(resource=resource)
+    logger_provider.add_log_record_processor(
+        BatchLogRecordProcessor(
+            OTLPLogExporter(endpoint=endpoint, insecure=insecure, timeout=3),
+            max_queue_size=2048,
+            schedule_delay_millis=5000,
+            max_export_batch_size=512,
+            export_timeout_millis=3000,
+        )
+    )
+    set_logger_provider(logger_provider)
+    logging_handler = LoggingHandler(level=logging.NOTSET, logger_provider=logger_provider)
+
    HTTPXClientInstrumentor().instrument()
    SQLAlchemyInstrumentor().instrument(enable_commenter=False)
-    _runtime = TelemetryRuntime(tracer_provider, meter_provider)
+    _runtime = TelemetryRuntime(
+        tracer_provider,
+        meter_provider,
+        logger_provider,
+        logging_handler,
+    )
    return _runtime


@@ -93,8 +122,8 @@ def instrument_fastapi(app: FastAPI) -> None:
 def add_trace_context(
    _logger: Any,
    _method_name: str,
-    event_dict: dict[str, Any],
-) -> dict[str, Any]:
+    event_dict: MutableMapping[str, Any],
+) -> MutableMapping[str, Any]:
    context = trace.get_current_span().get_span_context()
    if context.is_valid:
        event_dict["trace_id"] = format(context.trace_id, "032x")
@@ -10,6 +10,7 @@ from datetime import UTC, datetime, timedelta
 import httpx
 import structlog
 from opentelemetry import trace
+from opentelemetry.propagate import extract
 from prometheus_client import start_http_server
 from sqlalchemy import and_, func, or_, select, update

@@ -26,14 +27,27 @@ from app.metrics import (
 from app.provider import IdgtlClient, IdgtlConfig
 from app.service import RuntimeSettings, load_runtime_settings
 from app.settings import Settings, get_settings
-from app.telemetry import add_trace_context, init_telemetry
+from app.telemetry import TelemetryRuntime, add_trace_context, init_telemetry

 log = structlog.get_logger()
 MAX_CONNECT_ATTEMPTS = 3


-def configure_logging(level: str) -> None:
+def origin_links(traceparent: str | None) -> list[trace.Link]:
+    if not traceparent:
+        return []
+    context = extract({"traceparent": traceparent})
+    span_context = trace.get_current_span(context).get_span_context()
+    return [trace.Link(span_context)] if span_context.is_valid else []
+
+
+def configure_logging(level: str, telemetry: TelemetryRuntime | None = None) -> None:
    logging.basicConfig(level=level, format="%(message)s")
+    if telemetry and telemetry.logging_handler not in logging.getLogger().handlers:
+        telemetry.logging_handler.addFilter(
+            lambda record: not record.name.startswith("opentelemetry")
+        )
+        logging.getLogger().addHandler(telemetry.logging_handler)
    structlog.configure(
        processors=[
            structlog.contextvars.merge_contextvars,
@@ -42,7 +56,10 @@ def configure_logging(level: str) -> None:
            structlog.processors.TimeStamper(fmt="iso", utc=True, key="timestamp"),
            structlog.stdlib.add_log_level,
            structlog.processors.JSONRenderer(),
-        ]
+        ],
+        logger_factory=structlog.stdlib.LoggerFactory(),
+        wrapper_class=structlog.stdlib.BoundLogger,
+        cache_logger_on_first_use=True,
    )


@@ -177,7 +194,7 @@ async def update_queue_metrics(db: Database) -> None:
 async def worker_loop(stop: asyncio.Event) -> None:
    settings = get_settings()
    telemetry = init_telemetry("sms-worker")
-    configure_logging(settings.log_level)
+    configure_logging(settings.log_level, telemetry)
    structlog.contextvars.bind_contextvars(**{"service.name": "sms-worker"})
    metrics_server, metrics_thread = start_http_server(
        settings.metrics_port,
@@ -200,8 +217,17 @@ async def worker_loop(stop: asyncio.Event) -> None:
                        await update_queue_metrics(db)
                        await asyncio.wait_for(stop.wait(), timeout=runtime.poll_interval_ms / 1000)
                        continue
-                    process_span = tracer.start_span("sms.process", start_time=claim_started_ns)
-                    with trace.use_span(process_span, end_on_exit=True):
+                    process_span = tracer.start_span(
+                        "sms.process",
+                        start_time=claim_started_ns,
+                        links=origin_links(message.traceparent),
+                    )
+                    with (
+                        trace.use_span(process_span, end_on_exit=True),
+                        structlog.contextvars.bound_contextvars(
+                            sms_message_id=str(message.id)
+                        ),
+                    ):
                        claim_span = tracer.start_span("sms.claim", start_time=claim_started_ns)
                        claim_span.set_attribute("sms.claimed", True)
                        claim_span.end(end_time=claim_finished_ns)