Реализация на отдельных двух машинах с протестированным взаимодействием по проверке сообщений

2026-08-19 18:24:00 +03:00
parent bbef7a30c9
commit c7a80e7256
103 changed files with 3457 additions and 3725 deletions
@@ -254,11 +254,23 @@ class SafetyClient:
    async def ready(self) -> bool:
        try:
            response = await self.http.get(
-                f"{str(self.settings.message_safety_url).rstrip('/')}/health/ready",
+                f"{str(self.settings.message_safety_url).rstrip('/')}/internal/safety/status",
                timeout=2,
            )
-            return response.status_code == 200
-        except httpx.HTTPError:
+            if response.status_code != 200:
+                return False
+            body = response.json()
+            capabilities = body.get("capabilities", {})
+            return (
+                body.get("status") in {"ok", "degraded"}
+                and body.get("processing_mode") == "standard"
+                and type(body.get("config_version")) is int
+                and all(
+                    capabilities.get(name) == "ready"
+                    for name in ("text", "links", "files", "worker")
+                )
+            )
+        except (httpx.HTTPError, AttributeError, ValueError):
            return False


@@ -138,9 +138,7 @@ async def lifespan(app: FastAPI):
    app.state.settings = settings
    app.state.db = Database(settings.database_url)
    app.state.http = httpx.AsyncClient()
-    app.state.safety_http = httpx.AsyncClient(
-        verify=settings.message_safety_ca_file or True
-    )
+    app.state.safety_http = httpx.AsyncClient(verify=settings.message_safety_ca_file)
    app.state.redis = redis.from_url(settings.redis_url, decode_responses=True)
    app.state.redis_rt = redis.from_url(settings.redis_realtime_url, decode_responses=True)
    app.state.jwks = JWKSValidator(settings, app.state.http)
@@ -74,6 +74,49 @@ def safety_reply_message(dialog_id: uuid.UUID, content_kind: str) -> Message:
    )


+def safety_task_recovery_at(now: datetime, settings: Settings) -> datetime:
+    """Keep recovery behind the synchronous poller's worst-case final attempt."""
+    return now + timedelta(
+        seconds=settings.message_safety_task_poll_max_sec
+        + settings.message_safety_task_poll_interval_sec
+        + 5
+    )
+
+
+async def ensure_delivery_outbox(
+    session: AsyncSession,
+    message_id: uuid.UUID,
+    external_chat_id: uuid.UUID,
+    payload_json: dict[str, Any],
+    next_attempt_at: datetime,
+) -> DeliveryOutbox:
+    """Create the per-message outbox row or return the concurrent winner."""
+    statement = (
+        insert(DeliveryOutbox)
+        .values(
+            id=uuid.uuid4(),
+            message_id=message_id,
+            external_chat_id=external_chat_id,
+            payload_json=payload_json,
+            next_attempt_at=next_attempt_at,
+        )
+        .on_conflict_do_nothing(index_elements=[DeliveryOutbox.message_id])
+        .returning(DeliveryOutbox.id)
+    )
+    outbox_id = (await session.execute(statement)).scalar_one_or_none()
+    if outbox_id is not None:
+        outbox = await session.get(DeliveryOutbox, outbox_id)
+    else:
+        outbox = (
+            await session.execute(
+                select(DeliveryOutbox).where(DeliveryOutbox.message_id == message_id)
+            )
+        ).scalar_one()
+    if outbox is None:  # pragma: no cover - defensive guard for an invalid DB response
+        raise RuntimeError("Delivery outbox row was not returned")
+    return outbox
+
+
 class DomainError(Exception):
    def __init__(self, code: str, status: int, message: str, details: dict[str, Any] | None = None):
        self.code, self.status, self.message = code, status, message
@@ -918,7 +961,9 @@ async def send_message(
                expires_at=datetime.fromisoformat(verdict["expires_at"].replace("Z", "+00:00")),
                deadline_at=now
                + timedelta(seconds=settings.message_safety_task_poll_max_sec + 900),
-                next_poll_at=now,
+                # The request owns the first polling window. Recovery starts
+                # afterwards if the request crashes before completing the task.
+                next_poll_at=safety_task_recovery_at(datetime.now(UTC), settings),
            )
            session.add(task)
            await session.commit()
@@ -987,7 +1032,8 @@ async def send_message(
                "bypassed" if verdict["processing_mode"] == "mock" else "clean"
            )
        message.safety_status = "allowed"
-        outbox = DeliveryOutbox(
+        outbox = await ensure_delivery_outbox(
+            session,
            message_id=message.id,
            external_chat_id=dialog_id,
            payload_json={
@@ -1019,7 +1065,6 @@ async def send_message(
            next_attempt_at=datetime.now(UTC)
            + timedelta(seconds=settings.bitrix_local_app_http_timeout_sec + 5),
        )
-        session.add(outbox)
        await session.commit()
        await publish_message_status(fanout, message, settings)
        delivery_payload = await fresh_openlines_payload(outbox.payload_json, s3)
@@ -1,5 +1,7 @@
 import asyncio
 import uuid
+from collections.abc import AsyncIterator
+from contextlib import asynccontextmanager
 from datetime import UTC, datetime, timedelta

 import httpx
@@ -26,12 +28,28 @@ from app.integrations import (
 from app.notification_models import ClientUploadDraft
 from app.notification_service import expire_notifications
 from app.realtime import RealtimeFanout
-from app.services import load_settings, publish_dialog_status, publish_message_status
+from app.services import (
+    ensure_delivery_outbox,
+    load_settings,
+    publish_dialog_status,
+    publish_message_status,
+)
 from app.settings import Settings, get_settings

 log = structlog.get_logger()


+@asynccontextmanager
+async def worker_http_clients(
+    settings: Settings,
+) -> AsyncIterator[tuple[httpx.AsyncClient, httpx.AsyncClient]]:
+    async with (
+        httpx.AsyncClient() as openlines_http,
+        httpx.AsyncClient(verify=settings.message_safety_ca_file) as safety_http,
+    ):
+        yield openlines_http, safety_http
+
+
 async def delivery_once(
    db: Database,
    client: OpenLinesClient,
@@ -171,37 +189,38 @@ async def safety_once(
                        else None
                    )
                    if dialog and user:
-                        session.add(
-                            DeliveryOutbox(
-                                message_id=message.id,
-                                external_chat_id=dialog.id,
-                                payload_json={
-                                    "message_id": str(message.id),
-                                    "external_chat_id": str(dialog.id),
-                                    "occurred_at": message.occurred_at.isoformat(),
-                                    "user": {
-                                        "id": str(user.id),
-                                        "display_name": user.phone_number,
-                                    },
-                                    "message": {
-                                        "content_kind": message.content_kind,
-                                        "text": message.text,
-                                        "files": (
-                                            [{
+                        await ensure_delivery_outbox(
+                            session,
+                            message_id=message.id,
+                            external_chat_id=dialog.id,
+                            payload_json={
+                                "message_id": str(message.id),
+                                "external_chat_id": str(dialog.id),
+                                "occurred_at": message.occurred_at.isoformat(),
+                                "user": {
+                                    "id": str(user.id),
+                                    "display_name": user.phone_number,
+                                },
+                                "message": {
+                                    "content_kind": message.content_kind,
+                                    "text": message.text,
+                                    "files": (
+                                        [
+                                            {
                                                "attachment_id": str(attachment.id),
                                                "name": attachment.safe_file_name,
                                                "mime_type": attachment.mime_type,
                                                "size_bytes": attachment.size_bytes,
                                                "_storage_bucket": attachment.storage_bucket,
                                                "_object_key": attachment.object_key,
-                                            }]
-                                            if attachment
-                                            else []
-                                        ),
-                                    },
+                                            }
+                                        ]
+                                        if attachment
+                                        else []
+                                    ),
                                },
-                                next_attempt_at=datetime.now(UTC),
-                            )
+                            },
+                            next_attempt_at=datetime.now(UTC),
                        )
                elif verdict["_status"] == 403 and message:
                    message.safety_processing_mode = verdict["processing_mode"]
@@ -270,26 +289,25 @@ async def cleanup_once(db: Database, s3: S3Client, batch_size: int = 100) -> int
 async def loop(kind: str) -> None:
    settings = get_settings()
    db = Database(settings.database_url)
-    http = httpx.AsyncClient()
-    safety = SafetyClient(settings, http)
-    openlines = OpenLinesClient(settings, http)
    s3 = S3Client(settings)
    redis_rt = redis.from_url(settings.redis_realtime_url, decode_responses=True)
    fanout = RealtimeFanout(redis_rt)
    worker_id = f"{kind}-{uuid.uuid4()}"
    try:
-        while True:
-            count = 0
-            if kind == "delivery":
-                count = await delivery_once(db, openlines, s3, fanout, settings, worker_id)
-            elif kind == "safety":
-                count = await safety_once(db, safety, s3, fanout, settings, worker_id)
-            else:
-                count = await cleanup_once(db, s3)
-            if not count:
-                await asyncio.sleep(settings.worker_poll_interval_sec)
+        async with worker_http_clients(settings) as (openlines_http, safety_http):
+            safety = SafetyClient(settings, safety_http)
+            openlines = OpenLinesClient(settings, openlines_http)
+            while True:
+                count = 0
+                if kind == "delivery":
+                    count = await delivery_once(db, openlines, s3, fanout, settings, worker_id)
+                elif kind == "safety":
+                    count = await safety_once(db, safety, s3, fanout, settings, worker_id)
+                else:
+                    count = await cleanup_once(db, s3)
+                if not count:
+                    await asyncio.sleep(settings.worker_poll_interval_sec)
    finally:
-        await http.aclose()
        await redis_rt.aclose()
        await db.close()