Реализация на отдельных двух машинах с протестированным взаимодействием по проверке сообщений

This commit is contained in:
mi
2026-08-19 18:24:00 +03:00
parent bbef7a30c9
commit c7a80e7256
103 changed files with 3457 additions and 3725 deletions
@@ -14,6 +14,8 @@ RUN pip install --no-cache-dir /tmp/*.whl && rm -f /tmp/*.whl
COPY alembic.ini ./
COPY alembic ./alembic
COPY --chmod=0555 container-entrypoint.sh /usr/local/bin/han-container-entrypoint
RUN sed -i 's/\r$//' /usr/local/bin/han-container-entrypoint \
&& /bin/sh -n /usr/local/bin/han-container-entrypoint
USER 10001:10001
EXPOSE 8000
ENTRYPOINT ["/usr/local/bin/han-container-entrypoint"]
@@ -56,11 +56,12 @@ han-notification-draft-cleanup-worker
продюсера Notification Center; в БД синхронизируется только SHA-256 hash;
- `OTEL_EXPORTER_OTLP_ENDPOINT` — опциональный endpoint collector.
Токены генерируются `openssl rand -hex 32`. S3 read-only credentials Message Safety
не передаются этому контейнеру. В production подключение PostgreSQL должно использовать
TLS. Target `MESSAGE_SAFETY_URL=https://processing.internal:8443`; certificate
проверяется по internal CA, plaintext HTTP запрещён. Текущий Docker hostname
`message-safety` относится только к legacy stub до cutover.
Токены генерируются `openssl rand -hex 32`. `MESSAGE_SAFETY_SERVICE_TOKEN`
сохраняется как caller secret API backend; S3 credentials сервису Safety не
передаются. В production подключение PostgreSQL должно использовать TLS. Target
`MESSAGE_SAFETY_URL=https://processing.internal:8443`, API prefix
`/internal/safety/v2`; certificate проверяется по CA из
`MESSAGE_SAFETY_CA_HOST_PATH`, plaintext HTTP запрещён.
Smoke-сценарий `producer_test`: отправить `POST
/internal/notifications/v1/notifications` с `Authorization: Bearer
@@ -26,37 +26,70 @@ def upgrade() -> None:
ADD COLUMN IF NOT EXISTS last_error_code varchar(64),
ADD COLUMN IF NOT EXISTS last_error_at timestamptz,
ADD COLUMN IF NOT EXISTS completed_at timestamptz,
ADD COLUMN IF NOT EXISTS cancel_reason varchar(255);
ADD COLUMN IF NOT EXISTS cancel_reason varchar(255)
"""
)
op.execute(
"""
UPDATE han_app.sync_queue
SET status = CASE status
WHEN 'processing' THEN 'pending'
WHEN 'failed' THEN 'retry_wait'
ELSE status
END
WHERE status IN ('processing', 'failed');
WHERE status IN ('processing', 'failed')
"""
)
op.execute(
"""
ALTER TABLE han_app.sync_queue
DROP CONSTRAINT IF EXISTS sync_queue_status_check;
DROP CONSTRAINT IF EXISTS sync_queue_status_check
"""
)
op.execute(
"""
ALTER TABLE han_app.sync_queue
ADD CONSTRAINT sync_queue_status_check CHECK (
status IN ('pending','leased','processed','retry_wait','dead_letter','cancelled')
) NOT VALID;
) NOT VALID
"""
)
op.execute(
"""
ALTER TABLE han_app.sync_queue
VALIDATE CONSTRAINT sync_queue_status_check;
VALIDATE CONSTRAINT sync_queue_status_check
"""
)
op.execute(
"""
ALTER TABLE han_app.sync_queue
DROP CONSTRAINT IF EXISTS sync_queue_dedup_key_key;
DROP INDEX IF EXISTS han_app.ix_sync_queue_status_next;
DROP CONSTRAINT IF EXISTS sync_queue_dedup_key_key
"""
)
op.execute("DROP INDEX IF EXISTS han_app.ix_sync_queue_status_next")
op.execute(
"""
CREATE INDEX IF NOT EXISTS ix_sync_queue_claim
ON han_app.sync_queue(status, next_attempt_at, created_at);
ON han_app.sync_queue(status, next_attempt_at, created_at)
"""
)
op.execute(
"""
CREATE INDEX IF NOT EXISTS ix_sync_queue_expired_lease
ON han_app.sync_queue(locked_until) WHERE status = 'leased';
ON han_app.sync_queue(locked_until) WHERE status = 'leased'
"""
)
op.execute(
"""
CREATE INDEX IF NOT EXISTS ix_sync_queue_entity_history
ON han_app.sync_queue(entity_type, entity_id, created_at DESC);
ON han_app.sync_queue(entity_type, entity_id, created_at DESC)
"""
)
op.execute(
"""
CREATE UNIQUE INDEX IF NOT EXISTS uq_sync_queue_active_dedup
ON han_app.sync_queue(dedup_key)
WHERE status IN ('pending','leased','retry_wait');
WHERE status IN ('pending','leased','retry_wait')
"""
)
@@ -173,13 +206,20 @@ def upgrade() -> None:
updated_at = now();
RETURN NEW;
END;
$$;
$$
"""
)
op.execute(
"""
DROP TRIGGER IF EXISTS trg_profile_contact_sync ON han_app.client_profiles;
"""
)
op.execute(
"""
CREATE TRIGGER trg_profile_contact_sync
AFTER INSERT OR UPDATE OF record_status
ON han_app.client_profiles
FOR EACH ROW EXECUTE FUNCTION han_app.enqueue_contact_sync();
FOR EACH ROW EXECUTE FUNCTION han_app.enqueue_contact_sync()
"""
)
@@ -21,32 +21,58 @@ def upgrade() -> None:
ALTER TABLE han_app.messages
ADD COLUMN IF NOT EXISTS safety_processing_mode varchar(16),
ADD COLUMN IF NOT EXISTS safety_config_version bigint,
ADD COLUMN IF NOT EXISTS safety_rules_version varchar(128);
ADD COLUMN IF NOT EXISTS safety_rules_version varchar(128)
"""
)
op.execute(
"""
ALTER TABLE han_app.message_attachments
ADD COLUMN IF NOT EXISTS quarantine_version_id varchar(1024),
ADD COLUMN IF NOT EXISTS quarantine_etag varchar(1024);
ADD COLUMN IF NOT EXISTS quarantine_etag varchar(1024)
"""
)
op.execute(
"""
ALTER TABLE han_app.message_attachments
DROP CONSTRAINT IF EXISTS message_attachments_scan_status_check;
DROP CONSTRAINT IF EXISTS message_attachments_scan_status_check
"""
)
op.execute(
"""
ALTER TABLE han_app.message_attachments
ADD CONSTRAINT message_attachments_scan_status_check
CHECK (scan_status IN ('pending','clean','bypassed','infected','failed')) NOT VALID;
CHECK (scan_status IN ('pending','clean','bypassed','infected','failed')) NOT VALID
"""
)
op.execute(
"""
ALTER TABLE han_app.message_attachments
VALIDATE CONSTRAINT message_attachments_scan_status_check;
VALIDATE CONSTRAINT message_attachments_scan_status_check
"""
)
op.execute(
"""
ALTER TABLE han_app.safety_tasks
ADD COLUMN IF NOT EXISTS poll_location varchar(1024),
ADD COLUMN IF NOT EXISTS processing_mode varchar(16),
ADD COLUMN IF NOT EXISTS config_version bigint,
ADD COLUMN IF NOT EXISTS rules_version varchar(128),
ADD COLUMN IF NOT EXISTS expires_at timestamptz;
ADD COLUMN IF NOT EXISTS expires_at timestamptz
"""
)
op.execute(
"""
UPDATE han_app.safety_tasks
SET poll_location = '/internal/safety/v2/messages/tasks/' || task_id,
expires_at = deadline_at
WHERE poll_location IS NULL OR expires_at IS NULL;
WHERE poll_location IS NULL OR expires_at IS NULL
"""
)
op.execute(
"""
ALTER TABLE han_app.safety_tasks
ALTER COLUMN poll_location SET NOT NULL,
ALTER COLUMN expires_at SET NOT NULL;
ALTER COLUMN expires_at SET NOT NULL
"""
)
@@ -58,14 +84,26 @@ def upgrade() -> None:
ADD COLUMN IF NOT EXISTS quarantine_etag varchar(1024),
ADD COLUMN IF NOT EXISTS safety_processing_mode varchar(16),
ADD COLUMN IF NOT EXISTS safety_config_version bigint,
ADD COLUMN IF NOT EXISTS safety_rules_version varchar(128);
ADD COLUMN IF NOT EXISTS safety_rules_version varchar(128)
"""
)
op.execute(
"""
ALTER TABLE han_app.client_upload_drafts
DROP CONSTRAINT IF EXISTS client_upload_drafts_scan_status_check;
DROP CONSTRAINT IF EXISTS client_upload_drafts_scan_status_check
"""
)
op.execute(
"""
ALTER TABLE han_app.client_upload_drafts
ADD CONSTRAINT client_upload_drafts_scan_status_check
CHECK (scan_status IN ('pending','clean','bypassed','infected','failed')) NOT VALID;
CHECK (scan_status IN ('pending','clean','bypassed','infected','failed')) NOT VALID
"""
)
op.execute(
"""
ALTER TABLE han_app.client_upload_drafts
VALIDATE CONSTRAINT client_upload_drafts_scan_status_check;
VALIDATE CONSTRAINT client_upload_drafts_scan_status_check
"""
)
@@ -254,11 +254,23 @@ class SafetyClient:
async def ready(self) -> bool:
try:
response = await self.http.get(
f"{str(self.settings.message_safety_url).rstrip('/')}/health/ready",
f"{str(self.settings.message_safety_url).rstrip('/')}/internal/safety/status",
timeout=2,
)
return response.status_code == 200
except httpx.HTTPError:
if response.status_code != 200:
return False
body = response.json()
capabilities = body.get("capabilities", {})
return (
body.get("status") in {"ok", "degraded"}
and body.get("processing_mode") == "standard"
and type(body.get("config_version")) is int
and all(
capabilities.get(name) == "ready"
for name in ("text", "links", "files", "worker")
)
)
except (httpx.HTTPError, AttributeError, ValueError):
return False
@@ -138,9 +138,7 @@ async def lifespan(app: FastAPI):
app.state.settings = settings
app.state.db = Database(settings.database_url)
app.state.http = httpx.AsyncClient()
app.state.safety_http = httpx.AsyncClient(
verify=settings.message_safety_ca_file or True
)
app.state.safety_http = httpx.AsyncClient(verify=settings.message_safety_ca_file)
app.state.redis = redis.from_url(settings.redis_url, decode_responses=True)
app.state.redis_rt = redis.from_url(settings.redis_realtime_url, decode_responses=True)
app.state.jwks = JWKSValidator(settings, app.state.http)
@@ -74,6 +74,49 @@ def safety_reply_message(dialog_id: uuid.UUID, content_kind: str) -> Message:
)
def safety_task_recovery_at(now: datetime, settings: Settings) -> datetime:
"""Keep recovery behind the synchronous poller's worst-case final attempt."""
return now + timedelta(
seconds=settings.message_safety_task_poll_max_sec
+ settings.message_safety_task_poll_interval_sec
+ 5
)
async def ensure_delivery_outbox(
session: AsyncSession,
message_id: uuid.UUID,
external_chat_id: uuid.UUID,
payload_json: dict[str, Any],
next_attempt_at: datetime,
) -> DeliveryOutbox:
"""Create the per-message outbox row or return the concurrent winner."""
statement = (
insert(DeliveryOutbox)
.values(
id=uuid.uuid4(),
message_id=message_id,
external_chat_id=external_chat_id,
payload_json=payload_json,
next_attempt_at=next_attempt_at,
)
.on_conflict_do_nothing(index_elements=[DeliveryOutbox.message_id])
.returning(DeliveryOutbox.id)
)
outbox_id = (await session.execute(statement)).scalar_one_or_none()
if outbox_id is not None:
outbox = await session.get(DeliveryOutbox, outbox_id)
else:
outbox = (
await session.execute(
select(DeliveryOutbox).where(DeliveryOutbox.message_id == message_id)
)
).scalar_one()
if outbox is None: # pragma: no cover - defensive guard for an invalid DB response
raise RuntimeError("Delivery outbox row was not returned")
return outbox
class DomainError(Exception):
def __init__(self, code: str, status: int, message: str, details: dict[str, Any] | None = None):
self.code, self.status, self.message = code, status, message
@@ -918,7 +961,9 @@ async def send_message(
expires_at=datetime.fromisoformat(verdict["expires_at"].replace("Z", "+00:00")),
deadline_at=now
+ timedelta(seconds=settings.message_safety_task_poll_max_sec + 900),
next_poll_at=now,
# The request owns the first polling window. Recovery starts
# afterwards if the request crashes before completing the task.
next_poll_at=safety_task_recovery_at(datetime.now(UTC), settings),
)
session.add(task)
await session.commit()
@@ -987,7 +1032,8 @@ async def send_message(
"bypassed" if verdict["processing_mode"] == "mock" else "clean"
)
message.safety_status = "allowed"
outbox = DeliveryOutbox(
outbox = await ensure_delivery_outbox(
session,
message_id=message.id,
external_chat_id=dialog_id,
payload_json={
@@ -1019,7 +1065,6 @@ async def send_message(
next_attempt_at=datetime.now(UTC)
+ timedelta(seconds=settings.bitrix_local_app_http_timeout_sec + 5),
)
session.add(outbox)
await session.commit()
await publish_message_status(fanout, message, settings)
delivery_payload = await fresh_openlines_payload(outbox.payload_json, s3)
@@ -1,5 +1,7 @@
import asyncio
import uuid
from collections.abc import AsyncIterator
from contextlib import asynccontextmanager
from datetime import UTC, datetime, timedelta
import httpx
@@ -26,12 +28,28 @@ from app.integrations import (
from app.notification_models import ClientUploadDraft
from app.notification_service import expire_notifications
from app.realtime import RealtimeFanout
from app.services import load_settings, publish_dialog_status, publish_message_status
from app.services import (
ensure_delivery_outbox,
load_settings,
publish_dialog_status,
publish_message_status,
)
from app.settings import Settings, get_settings
log = structlog.get_logger()
@asynccontextmanager
async def worker_http_clients(
settings: Settings,
) -> AsyncIterator[tuple[httpx.AsyncClient, httpx.AsyncClient]]:
async with (
httpx.AsyncClient() as openlines_http,
httpx.AsyncClient(verify=settings.message_safety_ca_file) as safety_http,
):
yield openlines_http, safety_http
async def delivery_once(
db: Database,
client: OpenLinesClient,
@@ -171,37 +189,38 @@ async def safety_once(
else None
)
if dialog and user:
session.add(
DeliveryOutbox(
message_id=message.id,
external_chat_id=dialog.id,
payload_json={
"message_id": str(message.id),
"external_chat_id": str(dialog.id),
"occurred_at": message.occurred_at.isoformat(),
"user": {
"id": str(user.id),
"display_name": user.phone_number,
},
"message": {
"content_kind": message.content_kind,
"text": message.text,
"files": (
[{
await ensure_delivery_outbox(
session,
message_id=message.id,
external_chat_id=dialog.id,
payload_json={
"message_id": str(message.id),
"external_chat_id": str(dialog.id),
"occurred_at": message.occurred_at.isoformat(),
"user": {
"id": str(user.id),
"display_name": user.phone_number,
},
"message": {
"content_kind": message.content_kind,
"text": message.text,
"files": (
[
{
"attachment_id": str(attachment.id),
"name": attachment.safe_file_name,
"mime_type": attachment.mime_type,
"size_bytes": attachment.size_bytes,
"_storage_bucket": attachment.storage_bucket,
"_object_key": attachment.object_key,
}]
if attachment
else []
),
},
}
]
if attachment
else []
),
},
next_attempt_at=datetime.now(UTC),
)
},
next_attempt_at=datetime.now(UTC),
)
elif verdict["_status"] == 403 and message:
message.safety_processing_mode = verdict["processing_mode"]
@@ -270,26 +289,25 @@ async def cleanup_once(db: Database, s3: S3Client, batch_size: int = 100) -> int
async def loop(kind: str) -> None:
settings = get_settings()
db = Database(settings.database_url)
http = httpx.AsyncClient()
safety = SafetyClient(settings, http)
openlines = OpenLinesClient(settings, http)
s3 = S3Client(settings)
redis_rt = redis.from_url(settings.redis_realtime_url, decode_responses=True)
fanout = RealtimeFanout(redis_rt)
worker_id = f"{kind}-{uuid.uuid4()}"
try:
while True:
count = 0
if kind == "delivery":
count = await delivery_once(db, openlines, s3, fanout, settings, worker_id)
elif kind == "safety":
count = await safety_once(db, safety, s3, fanout, settings, worker_id)
else:
count = await cleanup_once(db, s3)
if not count:
await asyncio.sleep(settings.worker_poll_interval_sec)
async with worker_http_clients(settings) as (openlines_http, safety_http):
safety = SafetyClient(settings, safety_http)
openlines = OpenLinesClient(settings, openlines_http)
while True:
count = 0
if kind == "delivery":
count = await delivery_once(db, openlines, s3, fanout, settings, worker_id)
elif kind == "safety":
count = await safety_once(db, safety, s3, fanout, settings, worker_id)
else:
count = await cleanup_once(db, s3)
if not count:
await asyncio.sleep(settings.worker_poll_interval_sec)
finally:
await http.aclose()
await redis_rt.aclose()
await db.close()
@@ -27,8 +27,8 @@ services:
KEYCLOAK_INTERNAL_URL: ${KEYCLOAK_INTERNAL_URL}
KEYCLOAK_REALM: ${KEYCLOAK_REALM}
KEYCLOAK_AUDIENCE: ${KEYCLOAK_AUDIENCE}
MESSAGE_SAFETY_URL: ${MESSAGE_SAFETY_URL}
MESSAGE_SAFETY_API_PREFIX: /internal/safety/v2
MESSAGE_SAFETY_URL: ${MESSAGE_SAFETY_URL:-https://processing.internal:8443}
MESSAGE_SAFETY_API_PREFIX: ${MESSAGE_SAFETY_API_PREFIX:-/internal/safety/v2}
MESSAGE_SAFETY_CA_FILE: /run/config/message-safety-internal-ca.pem
MESSAGE_SAFETY_POST_TIMEOUT_SEC: ${MESSAGE_SAFETY_POST_TIMEOUT_SEC:-5}
MESSAGE_SAFETY_TASK_POLL_INTERVAL_SEC: ${MESSAGE_SAFETY_TASK_POLL_INTERVAL_SEC:-2}
@@ -13,6 +13,7 @@ from app.integrations import (
)
from app.realtime import CHANNEL_PREFIX
from app.settings import Settings
from app.workers import worker_http_clients
def settings() -> Settings:
@@ -25,8 +26,9 @@ def settings() -> Settings:
"KEYCLOAK_INTERNAL_URL": "http://keycloak:8080",
"KEYCLOAK_REALM": "han",
"KEYCLOAK_AUDIENCE": "api",
"MESSAGE_SAFETY_URL": "http://safety:8080",
"MESSAGE_SAFETY_URL": "https://processing.internal:8443",
"MESSAGE_SAFETY_SERVICE_TOKEN": "safety-token",
"MESSAGE_SAFETY_CA_FILE": "/run/config/message-safety-internal-ca.pem",
"BITRIX_LOCAL_APP_BASE_URL": "http://bitrix:8080",
"BITRIX_LOCAL_APP_INTERNAL_TOKEN": "bitrix-token",
"BITRIX_API_INBOX_TOKEN": "inbox-token",
@@ -42,6 +44,85 @@ def settings() -> Settings:
return Settings.model_validate(common)
@pytest.mark.asyncio
async def test_worker_uses_isolated_tls_client_for_remote_safety(monkeypatch) -> None:
created = []
class FakeAsyncClient:
def __init__(self, **kwargs):
self.kwargs = kwargs
self.closed = False
created.append(self)
async def __aenter__(self):
return self
async def __aexit__(self, *_):
self.closed = True
monkeypatch.setattr("app.workers.httpx.AsyncClient", FakeAsyncClient)
async with worker_http_clients(settings()) as (openlines_http, safety_http):
assert openlines_http.kwargs == {}
assert safety_http.kwargs == {
"verify": "/run/config/message-safety-internal-ca.pem"
}
assert openlines_http is not safety_http
assert all(client.closed for client in created)
@pytest.mark.asyncio
async def test_safety_ready_uses_private_status_alias_and_accepts_redis_degradation() -> None:
async def handler(request: httpx.Request) -> httpx.Response:
assert request.url.path == "/internal/safety/status"
return httpx.Response(
200,
json={
"status": "degraded",
"processing_mode": "standard",
"config_version": 1,
"capabilities": {
"text": "ready",
"links": "ready",
"files": "ready",
"worker": "ready",
},
},
)
async with httpx.AsyncClient(transport=httpx.MockTransport(handler)) as http:
assert await SafetyClient(settings(), http).ready() is True
@pytest.mark.asyncio
@pytest.mark.parametrize(
("processing_mode", "files"),
(("mock", "ready"), ("standard", "unavailable")),
)
async def test_safety_ready_rejects_unsafe_mode_or_unavailable_capability(
processing_mode: str, files: str
) -> None:
async def handler(_: httpx.Request) -> httpx.Response:
return httpx.Response(
200,
json={
"status": "degraded",
"processing_mode": processing_mode,
"config_version": 1,
"capabilities": {
"text": "ready",
"links": "ready",
"files": files,
"worker": "ready",
},
},
)
async with httpx.AsyncClient(transport=httpx.MockTransport(handler)) as http:
assert await SafetyClient(settings(), http).ready() is False
def test_realtime_channel_matches_redis_acl_namespace() -> None:
assert CHANNEL_PREFIX == "han:rt:dialog:"
@@ -1,4 +1,7 @@
import uuid
from datetime import UTC, datetime
from types import SimpleNamespace
from unittest.mock import AsyncMock
import pytest
from pydantic import TypeAdapter, ValidationError
@@ -14,7 +17,12 @@ from app.schemas import (
decode_cursor,
encode_cursor,
)
from app.services import MESSAGE_SAFETY_REPLIES, safety_reply_message
from app.services import (
MESSAGE_SAFETY_REPLIES,
ensure_delivery_outbox,
safety_reply_message,
safety_task_recovery_at,
)
def test_asyncpg_receives_libpq_dsn_without_sqlalchemy_driver() -> None:
@@ -67,6 +75,41 @@ def test_message_safety_business_replies_are_content_specific() -> None:
assert reply.delivery_status == "delivered"
def test_safety_recovery_starts_after_synchronous_polling_window() -> None:
now = datetime(2026, 8, 19, tzinfo=UTC)
settings = SimpleNamespace(
message_safety_task_poll_max_sec=300,
message_safety_task_poll_interval_sec=2,
)
assert (safety_task_recovery_at(now, settings) - now).total_seconds() == 307
async def test_delivery_outbox_returns_concurrent_insert_winner() -> None:
existing = object()
session = SimpleNamespace(
execute=AsyncMock(
side_effect=[
SimpleNamespace(scalar_one_or_none=lambda: None),
SimpleNamespace(scalar_one=lambda: existing),
]
),
get=AsyncMock(),
)
result = await ensure_delivery_outbox(
session,
message_id=uuid.uuid4(),
external_chat_id=uuid.uuid4(),
payload_json={"message_id": "test"},
next_attempt_at=datetime(2026, 8, 19, tzinfo=UTC),
)
assert result is existing
assert session.execute.await_count == 2
session.get.assert_not_awaited()
def test_fingerprint_is_canonical_and_user_scoped() -> None:
user = uuid.uuid4()
first = canonical_fingerprint("post", "/dialogs/{id}", {"id": "1"}, {"b": 2, "a": 1}, user)