ВМ1: реализован сбор логов

2026-09-07 12:46:45 +03:00
parent 44db38f6fe
commit 85df788f2d
42 changed files with 1669 additions and 310 deletions
@@ -1,12 +1,14 @@
 import asyncio
+import logging
 import uuid
-from collections.abc import AsyncIterator
+from collections.abc import AsyncIterator, Awaitable, Callable
 from contextlib import asynccontextmanager
 from datetime import UTC, datetime, timedelta

 import httpx
 import redis.asyncio as redis
 import structlog
+from opentelemetry import trace
 from sqlalchemy import delete, select

 from app.db import (
@@ -25,6 +27,7 @@ from app.integrations import (
    SafetyClient,
    fresh_openlines_payload,
 )
+from app.logging_security import redact_event
 from app.notification_models import ClientUploadDraft
 from app.notification_service import expire_notifications
 from app.realtime import RealtimeFanout
@@ -37,10 +40,45 @@ from app.services import (
    publish_message_status,
 )
 from app.settings import Settings, get_settings
+from app.telemetry import TelemetryRuntime, add_trace_context, init_telemetry, origin_links

 log = structlog.get_logger()


+def configure_logging(level: str, telemetry: TelemetryRuntime | None = None) -> None:
+    logging.basicConfig(level=level, format="%(message)s")
+    if telemetry and telemetry.logging_handler not in logging.getLogger().handlers:
+        telemetry.logging_handler.addFilter(
+            lambda record: not record.name.startswith("opentelemetry")
+        )
+        logging.getLogger().addHandler(telemetry.logging_handler)
+    structlog.configure(
+        processors=[
+            structlog.contextvars.merge_contextvars,
+            add_trace_context,
+            redact_event,
+            structlog.processors.TimeStamper(fmt="iso", utc=True, key="timestamp"),
+            structlog.stdlib.add_log_level,
+            structlog.processors.JSONRenderer(),
+        ],
+        logger_factory=structlog.stdlib.LoggerFactory(),
+        wrapper_class=structlog.stdlib.BoundLogger,
+        cache_logger_on_first_use=True,
+    )
+
+
+def run_worker(service_name: str, target: Callable[[], Awaitable[None]]) -> None:
+    settings = get_settings()
+    telemetry = init_telemetry(service_name)
+    configure_logging(settings.log_level, telemetry)
+    structlog.contextvars.bind_contextvars(**{"service.name": service_name})
+    try:
+        asyncio.run(target())
+    finally:
+        if telemetry:
+            telemetry.shutdown()
+
+
@asynccontextmanager
 async def worker_http_clients(
    settings: Settings,
@@ -61,6 +99,7 @@ async def delivery_once(
    worker_id: str,
    batch_size: int = 20,
 ) -> int:
+    tracer = trace.get_tracer("han.api.delivery-worker")
    async with db.sessions() as session:
        rows = (
            (
@@ -88,33 +127,43 @@ async def delivery_once(
            row = await session.get(DeliveryOutbox, row_id, with_for_update=True)
            if row is None:
                continue
-            message = None
-            dialog = None
-            try:
-                payload = await fresh_openlines_payload(row.payload_json, s3)
-                await client.send(row.message_id, payload, f"worker-{worker_id}")
-                row.status = "delivered"
-                message = await session.get(Message, row.message_id)
+            with (
+                tracer.start_as_current_span(
+                    "delivery.process",
+                    links=origin_links(row.traceparent),
+                ),
+                structlog.contextvars.bound_contextvars(
+                    delivery_outbox_id=str(row.id),
+                    message_id=str(row.message_id),
+                ),
+            ):
+                message = None
+                dialog = None
+                try:
+                    payload = await fresh_openlines_payload(row.payload_json, s3)
+                    await client.send(row.message_id, payload, f"worker-{worker_id}")
+                    row.status = "delivered"
+                    message = await session.get(Message, row.message_id)
+                    if message:
+                        message.delivery_status = "delivered"
+                        dialog = await session.get(Dialog, message.dialog_id)
+                        if dialog:
+                            dialog.status = "waiting_for_company"
+                            dialog.last_message_at = datetime.now(UTC)
+                except DependencyFailure:
+                    row.attempt_count += 1
+                    row.status = "dead_letter" if row.attempt_count >= 12 else "retry"
+                    row.next_attempt_at = datetime.now(UTC) + timedelta(
+                        seconds=min(3600, 2**row.attempt_count)
+                    )
+                    row.last_error_code = "dependency_unavailable"
+                row.locked_at = None
+                row.locked_by = None
+                await session.commit()
                if message:
-                    message.delivery_status = "delivered"
-                    dialog = await session.get(Dialog, message.dialog_id)
-                    if dialog:
-                        dialog.status = "waiting_for_company"
-                        dialog.last_message_at = datetime.now(UTC)
-            except DependencyFailure:
-                row.attempt_count += 1
-                row.status = "dead_letter" if row.attempt_count >= 12 else "retry"
-                row.next_attempt_at = datetime.now(UTC) + timedelta(
-                    seconds=min(3600, 2**row.attempt_count)
-                )
-                row.last_error_code = "dependency_unavailable"
-            row.locked_at = None
-            row.locked_by = None
-            await session.commit()
-            if message:
-                await publish_message_status(fanout, message, settings)
-            if dialog:
-                await publish_dialog_status(fanout, dialog)
+                    await publish_message_status(fanout, message, settings)
+                if dialog:
+                    await publish_dialog_status(fanout, dialog)
    return len(ids)


@@ -127,6 +176,7 @@ async def safety_once(
    worker_id: str,
    batch_size: int = 20,
 ) -> int:
+    tracer = trace.get_tracer("han.api.safety-recovery-worker")
    async with db.sessions() as session:
        rows = (
            (
@@ -155,7 +205,14 @@ async def safety_once(
                continue
            message = None
            try:
-                verdict = await safety.poll(task.poll_location, f"worker-{worker_id}")
+                with tracer.start_as_current_span(
+                    "safety.recovery.poll",
+                    links=origin_links(task.traceparent),
+                ):
+                    verdict = await safety.poll(
+                        task.poll_location,
+                        f"worker-{worker_id}",
+                    )
                message = await session.get(Message, task.message_id)
                attachment = (
                    await session.get(MessageAttachment, task.attachment_id)
@@ -204,6 +261,7 @@ async def safety_once(
                                attachment=attachment,
                            ),
                            next_attempt_at=datetime.now(UTC),
+                            traceparent=task.traceparent,
                        )
                elif verdict["_status"] == 403 and message:
                    message.safety_processing_mode = verdict["processing_mode"]
@@ -365,20 +423,20 @@ async def notification_draft_cleanup_loop() -> None:


 def delivery_main() -> None:
-    asyncio.run(loop("delivery"))
+    run_worker("delivery-worker", lambda: loop("delivery"))


 def safety_main() -> None:
-    asyncio.run(loop("safety"))
+    run_worker("safety-recovery-worker", lambda: loop("safety"))


 def cleanup_main() -> None:
-    asyncio.run(loop("cleanup"))
+    run_worker("cleanup-worker", lambda: loop("cleanup"))


 def notification_expire_main() -> None:
-    asyncio.run(notification_expire_loop())
+    run_worker("notification-expire-worker", notification_expire_loop)


 def notification_draft_cleanup_main() -> None:
-    asyncio.run(notification_draft_cleanup_loop())
+    run_worker("notification-draft-cleanup-worker", notification_draft_cleanup_loop)