mi
1230 lines
43 KiB
Python
1230 lines
43 KiB
Python
import hashlib
|
|
import json
|
|
import re
|
|
import unicodedata
|
|
import uuid
|
|
from dataclasses import dataclass
|
|
from datetime import UTC, datetime, timedelta
|
|
from pathlib import PurePath
|
|
from typing import Any
|
|
|
|
import httpx
|
|
from sqlalchemy import case, func, select
|
|
from sqlalchemy.dialects.postgresql import insert
|
|
from sqlalchemy.ext.asyncio import AsyncSession
|
|
|
|
from app.auth import Principal
|
|
from app.chat_settings import CHAT_MESSAGE_MAX_LENGTH_KEY, validate_chat_settings
|
|
from app.db import (
|
|
AppSetting,
|
|
AuditEvent,
|
|
ClientProfile,
|
|
DeliveryOutbox,
|
|
Dialog,
|
|
Document,
|
|
IdempotencyRecord,
|
|
Message,
|
|
MessageAttachment,
|
|
OpenLinesInboxReceipt,
|
|
SafetyTask,
|
|
UserConsent,
|
|
UserIdentity,
|
|
UxSession,
|
|
)
|
|
from app.integrations import (
|
|
DependencyFailure,
|
|
OpenLinesClient,
|
|
S3Client,
|
|
SafetyClient,
|
|
fresh_openlines_payload,
|
|
)
|
|
from app.otp_settings import OTP_SETTING_KEYS, validate_otp_settings
|
|
from app.realtime import RealtimeFanout
|
|
from app.schemas import (
|
|
AttachmentCompleteRequest,
|
|
AttachmentInitRequest,
|
|
BootstrapRequest,
|
|
ConsentsRequest,
|
|
FileMessageRequest,
|
|
MessageRequest,
|
|
OpenLinesInbox,
|
|
SessionStartRequest,
|
|
encode_cursor,
|
|
)
|
|
from app.settings import Settings
|
|
|
|
MESSAGE_SAFETY_REPLIES = {
|
|
"text": (
|
|
"К сожалению, ваше сообщение не соответствует правилам данного чата "
|
|
"и не может быть отправлено. Попробуйте переформулировать."
|
|
),
|
|
"file": "К сожалению, ваш документ не прошел проверку и не может быть доставлен.",
|
|
}
|
|
|
|
|
|
def safety_reply_message(dialog_id: uuid.UUID, content_kind: str) -> Message:
|
|
return Message(
|
|
dialog_id=dialog_id,
|
|
sender_type="company",
|
|
content_kind="text",
|
|
text=MESSAGE_SAFETY_REPLIES[content_kind],
|
|
safety_status="allowed",
|
|
delivery_status="delivered",
|
|
occurred_at=datetime.now(UTC),
|
|
)
|
|
|
|
|
|
class DomainError(Exception):
|
|
def __init__(self, code: str, status: int, message: str, details: dict[str, Any] | None = None):
|
|
self.code, self.status, self.message = code, status, message
|
|
self.details = details or {}
|
|
|
|
|
|
REQUIRED_SETTINGS = {
|
|
"auth.phone.enabled",
|
|
"auth.password.enabled",
|
|
"otp.phone.max_send_attempts_per_24h",
|
|
"otp.phone.min_seconds_between_attempts",
|
|
"otp.phone.max_verify_attempts",
|
|
"operator.call.phone",
|
|
"consent.personal_data.required",
|
|
"consent.personal_data.document_url",
|
|
"consent.personal_data.version",
|
|
"consent.privacy_policy.document_url",
|
|
"consent.user_agreement.required",
|
|
"consent.user_agreement.document_url",
|
|
"consent.user_agreement.version",
|
|
"consent.marketing.required",
|
|
"consent.marketing.document_url",
|
|
"consent.marketing.version",
|
|
"chat.attachments.allowed_extensions",
|
|
"chat.attachments.allowed_mime_types",
|
|
"chat.attachments.max_size_mb",
|
|
"chat.attachments.presigned_upload_ttl_seconds",
|
|
"rate_limit.message_send.per_user",
|
|
"rate_limit.message_send.per_dialog",
|
|
"rate_limit.download_url.per_user",
|
|
"rate_limit.public_endpoints.per_ip",
|
|
"ux.session.idle_timeout_minutes",
|
|
"security.cors.allowed_origins",
|
|
"security.public_cache.max_age_seconds",
|
|
"notification.home.max_items",
|
|
"notification.center.max_items",
|
|
"notification.carousel.autoplay_enabled",
|
|
"notification.carousel.autoplay_interval_ms",
|
|
"notification.hidden.default_ttl_days",
|
|
"notification.documents.max_files",
|
|
"notification.instruction.allowed_hosts",
|
|
"notification.expire_job.run_at",
|
|
"notification.upload_draft.ttl_days",
|
|
"rate_limit.notifications_read.per_user",
|
|
"rate_limit.notifications_action.per_user",
|
|
"rate_limit.notification_upload.per_user",
|
|
"rate_limit.notifications_public.per_ip",
|
|
CHAT_MESSAGE_MAX_LENGTH_KEY,
|
|
} | OTP_SETTING_KEYS
|
|
|
|
|
|
@dataclass(frozen=True, slots=True)
|
|
class SettingsSnapshot:
|
|
values: dict[str, str]
|
|
version: str
|
|
|
|
def boolean(self, key: str) -> bool:
|
|
return self.values[key].lower() == "true"
|
|
|
|
def integer(self, key: str) -> int:
|
|
return int(self.values[key])
|
|
|
|
def strings(self, key: str) -> list[str]:
|
|
return [item.strip() for item in self.values[key].split(",") if item.strip()]
|
|
|
|
def limit(self, key: str) -> tuple[int, int]:
|
|
amount, period = self.values[key].split("/", 1)
|
|
windows = {"second": 1, "minute": 60, "hour": 3600, "day": 86400}
|
|
return int(amount), windows[period]
|
|
|
|
|
|
@dataclass(frozen=True, slots=True)
|
|
class AuditContext:
|
|
request_id: str
|
|
trace_id: str
|
|
ux_session_id: uuid.UUID | None
|
|
user_agent_hash: str | None
|
|
client_ip: str | None
|
|
|
|
def with_ux_session(self, ux_session_id: uuid.UUID) -> "AuditContext":
|
|
return AuditContext(
|
|
request_id=self.request_id,
|
|
trace_id=self.trace_id,
|
|
ux_session_id=ux_session_id,
|
|
user_agent_hash=self.user_agent_hash,
|
|
client_ip=self.client_ip,
|
|
)
|
|
|
|
|
|
async def load_settings(session: AsyncSession) -> SettingsSnapshot:
|
|
rows = list(
|
|
(
|
|
await session.execute(select(AppSetting).where(AppSetting.record_status == "A"))
|
|
).scalars()
|
|
)
|
|
values = {row.setting_key: row.setting_value for row in rows}
|
|
missing = REQUIRED_SETTINGS - values.keys()
|
|
if missing:
|
|
raise DomainError(
|
|
"dependency_unavailable",
|
|
503,
|
|
"Required settings are unavailable",
|
|
{"missing": sorted(missing)},
|
|
)
|
|
try:
|
|
invalid_metadata = sorted(
|
|
row.setting_key
|
|
for row in rows
|
|
if row.setting_key in OTP_SETTING_KEYS
|
|
and (row.value_type != "integer" or row.is_public)
|
|
)
|
|
if invalid_metadata:
|
|
raise ValueError(
|
|
f"OTP settings must have integer type and be private: {invalid_metadata}"
|
|
)
|
|
validate_otp_settings(values)
|
|
validate_chat_settings(values)
|
|
except ValueError as error:
|
|
raise DomainError(
|
|
"dependency_unavailable",
|
|
503,
|
|
"Application settings are invalid",
|
|
{"reason": str(error)},
|
|
) from error
|
|
version = hashlib.sha256(json.dumps(values, sort_keys=True).encode()).hexdigest()[:24]
|
|
return SettingsSnapshot(values, version)
|
|
|
|
|
|
async def resolve_user(session: AsyncSession, principal: Principal) -> UserIdentity:
|
|
user = (
|
|
await session.execute(
|
|
select(UserIdentity).where(
|
|
UserIdentity.keycloak_sub == principal.subject, UserIdentity.record_status == "A"
|
|
)
|
|
)
|
|
).scalar_one_or_none()
|
|
if user is None:
|
|
raise DomainError("resource_state_conflict", 409, "Bootstrap required")
|
|
return user
|
|
|
|
|
|
def audit(
|
|
event_type: str,
|
|
context: AuditContext,
|
|
user_id: uuid.UUID | None,
|
|
resource_type: str | None = None,
|
|
resource_id: uuid.UUID | None = None,
|
|
metadata: dict[str, Any] | None = None,
|
|
outcome: str = "success",
|
|
) -> AuditEvent:
|
|
return AuditEvent(
|
|
event_type=event_type,
|
|
actor_type="user" if user_id else "service",
|
|
user_id=user_id,
|
|
ux_session_id=context.ux_session_id,
|
|
request_id=context.request_id,
|
|
trace_id=context.trace_id,
|
|
resource_type=resource_type,
|
|
resource_id=resource_id,
|
|
user_agent_hash=context.user_agent_hash,
|
|
outcome=outcome,
|
|
metadata_json=metadata or {},
|
|
)
|
|
|
|
|
|
def device_snapshot(device: Any) -> dict[str, str | None]:
|
|
return {
|
|
"platform": device.platform,
|
|
"app_version": device.app_version,
|
|
"device_id": device.device_id,
|
|
}
|
|
|
|
|
|
def consent_versions(consents: Any) -> dict[str, str]:
|
|
return {
|
|
name: getattr(consents, name).version
|
|
for name in ("personal_data", "user_agreement", "marketing")
|
|
}
|
|
|
|
|
|
def validate_consents(consents: Any, snapshot: SettingsSnapshot) -> None:
|
|
for name in ("personal_data", "user_agreement", "marketing"):
|
|
choice = getattr(consents, name)
|
|
if choice.version != snapshot.values[f"consent.{name}.version"]:
|
|
raise DomainError(
|
|
"validation_error", 400, "Consent version is not current", {"field": name}
|
|
)
|
|
if snapshot.boolean(f"consent.{name}.required") and not choice.accepted:
|
|
raise DomainError("consents_required", 403, "Required consents must be accepted")
|
|
|
|
|
|
async def bootstrap(
|
|
session: AsyncSession,
|
|
principal: Principal,
|
|
body: BootstrapRequest,
|
|
snapshot: SettingsSnapshot,
|
|
context: AuditContext,
|
|
) -> dict[str, Any]:
|
|
if not principal.phone_number:
|
|
raise DomainError("phone_claim_missing", 400, "Verified phone claim is missing")
|
|
validate_consents(body.consents, snapshot)
|
|
now = datetime.now(UTC)
|
|
device = device_snapshot(body.device)
|
|
statement = (
|
|
insert(UserIdentity)
|
|
.values(
|
|
id=uuid.uuid4(),
|
|
keycloak_sub=principal.subject,
|
|
phone_number=principal.phone_number,
|
|
last_login_at=now,
|
|
)
|
|
.on_conflict_do_update(
|
|
index_elements=[UserIdentity.keycloak_sub],
|
|
set_={"phone_number": principal.phone_number, "last_login_at": now, "updated_at": now},
|
|
)
|
|
.returning(UserIdentity.id)
|
|
)
|
|
user_id = (await session.execute(statement)).scalar_one()
|
|
await session.execute(
|
|
insert(ClientProfile)
|
|
.values(id=uuid.uuid4(), user_id=user_id, russian_phone=principal.phone_number)
|
|
.on_conflict_do_nothing(index_elements=[ClientProfile.user_id])
|
|
)
|
|
for consent_type in ("personal_data", "user_agreement", "marketing"):
|
|
choice = getattr(body.consents, consent_type)
|
|
await session.execute(
|
|
insert(UserConsent)
|
|
.values(
|
|
id=uuid.uuid4(),
|
|
user_id=user_id,
|
|
consent_type=consent_type,
|
|
document_version=choice.version,
|
|
accepted=choice.accepted,
|
|
accepted_at=now,
|
|
client_ip=context.client_ip,
|
|
user_agent_hash=context.user_agent_hash,
|
|
device_json=device,
|
|
)
|
|
.on_conflict_do_nothing(
|
|
index_elements=[
|
|
UserConsent.user_id,
|
|
UserConsent.consent_type,
|
|
UserConsent.document_version,
|
|
]
|
|
)
|
|
)
|
|
session.add(
|
|
audit(
|
|
"auth.bootstrap",
|
|
context,
|
|
user_id,
|
|
metadata={
|
|
"consent_versions": consent_versions(body.consents),
|
|
"platform": body.device.platform,
|
|
"app_version": body.device.app_version,
|
|
},
|
|
)
|
|
)
|
|
await session.commit()
|
|
return {"user_id": user_id, "profile_ready": True}
|
|
|
|
|
|
async def record_consents(
|
|
session: AsyncSession,
|
|
user: UserIdentity,
|
|
body: ConsentsRequest,
|
|
snapshot: SettingsSnapshot,
|
|
context: AuditContext,
|
|
) -> dict[str, Any]:
|
|
validate_consents(body.consents, snapshot)
|
|
if context.ux_session_id is None:
|
|
raise DomainError("validation_error", 400, "X-Ux-Session-Id is required")
|
|
ux_session = (
|
|
await session.execute(
|
|
select(UxSession).where(
|
|
UxSession.id == context.ux_session_id,
|
|
UxSession.user_id == user.id,
|
|
)
|
|
)
|
|
).scalar_one_or_none()
|
|
if ux_session is None:
|
|
raise DomainError("validation_error", 400, "X-Ux-Session-Id is invalid")
|
|
device = {
|
|
"platform": ux_session.platform,
|
|
"app_version": ux_session.app_version,
|
|
"device_id": ux_session.device_id,
|
|
}
|
|
now = datetime.now(UTC)
|
|
versions: dict[str, str] = {}
|
|
for consent_type in ("personal_data", "user_agreement", "marketing"):
|
|
choice = getattr(body.consents, consent_type)
|
|
versions[consent_type] = choice.version
|
|
statement = insert(UserConsent).values(
|
|
id=uuid.uuid4(),
|
|
user_id=user.id,
|
|
ux_session_id=context.ux_session_id,
|
|
consent_type=consent_type,
|
|
document_version=choice.version,
|
|
accepted=choice.accepted,
|
|
accepted_at=now,
|
|
client_ip=context.client_ip,
|
|
user_agent_hash=context.user_agent_hash,
|
|
device_json=device,
|
|
)
|
|
await session.execute(
|
|
statement.on_conflict_do_update(
|
|
index_elements=[
|
|
UserConsent.user_id,
|
|
UserConsent.consent_type,
|
|
UserConsent.document_version,
|
|
],
|
|
set_={
|
|
"ux_session_id": statement.excluded.ux_session_id,
|
|
"client_ip": func.coalesce(
|
|
UserConsent.client_ip, statement.excluded.client_ip
|
|
),
|
|
"user_agent_hash": func.coalesce(
|
|
UserConsent.user_agent_hash,
|
|
statement.excluded.user_agent_hash,
|
|
),
|
|
"device_json": case(
|
|
(UserConsent.device_json == {}, statement.excluded.device_json),
|
|
else_=UserConsent.device_json,
|
|
),
|
|
},
|
|
)
|
|
)
|
|
session.add(
|
|
audit(
|
|
"consent.recorded",
|
|
context,
|
|
user.id,
|
|
metadata={"consent_versions": versions},
|
|
)
|
|
)
|
|
await session.commit()
|
|
return {"recorded_at": now, "versions": versions}
|
|
|
|
|
|
async def start_session(
|
|
session: AsyncSession, user: UserIdentity, body: SessionStartRequest, context: AuditContext
|
|
) -> dict[str, Any]:
|
|
now, session_id = datetime.now(UTC), uuid.uuid4()
|
|
device = device_snapshot(body.device)
|
|
session.add(
|
|
UxSession(
|
|
id=session_id,
|
|
user_id=user.id,
|
|
start_reason=body.start_reason,
|
|
platform=body.device.platform,
|
|
app_version=body.device.app_version,
|
|
device_id=device["device_id"],
|
|
started_at=now,
|
|
)
|
|
)
|
|
session.add(
|
|
audit(
|
|
"session_start",
|
|
context.with_ux_session(session_id),
|
|
user.id,
|
|
metadata={
|
|
"start_reason": body.start_reason,
|
|
"platform": body.device.platform,
|
|
"app_version": body.device.app_version,
|
|
},
|
|
)
|
|
)
|
|
await session.commit()
|
|
return {"ux_session_id": session_id, "started_at": now}
|
|
|
|
|
|
async def get_profile(session: AsyncSession, user: UserIdentity) -> dict[str, Any]:
|
|
profile = (
|
|
await session.execute(
|
|
select(ClientProfile).where(
|
|
ClientProfile.user_id == user.id, ClientProfile.record_status == "A"
|
|
)
|
|
)
|
|
).scalar_one()
|
|
document_count = (
|
|
await session.scalar(
|
|
select(func.count(Document.id)).where(
|
|
Document.user_id == user.id, Document.record_status == "A"
|
|
)
|
|
)
|
|
or 0
|
|
)
|
|
return {
|
|
"user_id": user.id,
|
|
"profile": {
|
|
"personal_data": {
|
|
"full_name": profile.full_name,
|
|
"citizenship": profile.citizenship,
|
|
"russian_phone": profile.russian_phone,
|
|
"foreign_phone": profile.foreign_phone,
|
|
"email": profile.email,
|
|
},
|
|
"documents": {"count": document_count},
|
|
},
|
|
}
|
|
|
|
|
|
def dialog_dto(dialog: Dialog) -> dict[str, Any]:
|
|
return {
|
|
"dialog_id": dialog.id,
|
|
"status": dialog.status,
|
|
"last_message_preview": None,
|
|
"unread_count": 0,
|
|
"created_at": dialog.created_at,
|
|
"updated_at": dialog.updated_at,
|
|
}
|
|
|
|
|
|
def message_dto(
|
|
message: Message, attachments: list[MessageAttachment] | None = None
|
|
) -> dict[str, Any]:
|
|
return {
|
|
"message_id": message.id,
|
|
"dialog_id": message.dialog_id,
|
|
"sender_type": message.sender_type,
|
|
"content_kind": message.content_kind,
|
|
"text": message.text,
|
|
"attachments": [
|
|
{
|
|
"attachment_id": item.id,
|
|
"file_name": item.safe_file_name,
|
|
"mime_type": item.mime_type,
|
|
"size_bytes": item.size_bytes,
|
|
"scan_status": item.scan_status,
|
|
}
|
|
for item in (attachments or [])
|
|
],
|
|
"safety_status": message.safety_status,
|
|
"delivery_status": message.delivery_status,
|
|
"created_at": message.created_at,
|
|
}
|
|
|
|
|
|
def message_cursor(message: Message, settings: Settings) -> str:
|
|
return encode_cursor(
|
|
{"created_at": message.created_at.isoformat(), "id": str(message.id)},
|
|
settings.cursor_hmac_secret.get_secret_value().encode(),
|
|
)
|
|
|
|
|
|
async def publish_message(
|
|
fanout: RealtimeFanout,
|
|
message: Message,
|
|
settings: Settings,
|
|
attachments: list[MessageAttachment] | None = None,
|
|
) -> None:
|
|
await fanout.publish(
|
|
{
|
|
"type": "message.new",
|
|
"dialog_id": str(message.dialog_id),
|
|
"message": message_dto(message, attachments),
|
|
"cursor": message_cursor(message, settings),
|
|
}
|
|
)
|
|
|
|
|
|
async def publish_message_status(
|
|
fanout: RealtimeFanout, message: Message, settings: Settings
|
|
) -> None:
|
|
await fanout.publish(
|
|
{
|
|
"type": "message.status",
|
|
"dialog_id": str(message.dialog_id),
|
|
"message_id": str(message.id),
|
|
"safety_status": message.safety_status,
|
|
"delivery_status": message.delivery_status,
|
|
"cursor": message_cursor(message, settings),
|
|
}
|
|
)
|
|
|
|
|
|
async def publish_dialog_status(fanout: RealtimeFanout, dialog: Dialog) -> None:
|
|
await fanout.publish(
|
|
{"type": "dialog.status", "dialog_id": str(dialog.id), "status": dialog.status}
|
|
)
|
|
|
|
|
|
async def owned_dialog(session: AsyncSession, user_id: uuid.UUID, dialog_id: uuid.UUID) -> Dialog:
|
|
dialog = (
|
|
await session.execute(
|
|
select(Dialog).where(
|
|
Dialog.id == dialog_id, Dialog.user_id == user_id, Dialog.record_status == "A"
|
|
)
|
|
)
|
|
).scalar_one_or_none()
|
|
if dialog is None:
|
|
raise DomainError("not_found", 404, "Resource was not found")
|
|
return dialog
|
|
|
|
|
|
async def create_dialog(
|
|
session: AsyncSession, user: UserIdentity, context: AuditContext, idempotency_key: str
|
|
) -> tuple[dict[str, Any], int]:
|
|
scope = "dialogs.create"
|
|
fingerprint = hashlib.sha256(f"POST:/api/v1/dialogs:{user.id}".encode()).hexdigest()
|
|
record = (
|
|
await session.execute(
|
|
select(IdempotencyRecord).where(
|
|
IdempotencyRecord.scope == scope,
|
|
IdempotencyRecord.user_id == user.id,
|
|
IdempotencyRecord.idempotency_key == idempotency_key,
|
|
)
|
|
)
|
|
).scalar_one_or_none()
|
|
if record:
|
|
if record.request_fingerprint != fingerprint:
|
|
raise DomainError("idempotency_key_reused", 409, "Idempotency key was reused")
|
|
if record.status == "completed" and record.response_body_json:
|
|
return record.response_body_json, record.response_status or 200
|
|
existing = (
|
|
await session.execute(
|
|
select(Dialog).where(
|
|
Dialog.user_id == user.id,
|
|
Dialog.record_status == "A",
|
|
Dialog.status.in_(["open", "waiting_for_company", "waiting_for_client"]),
|
|
)
|
|
)
|
|
).scalar_one_or_none()
|
|
if existing:
|
|
result = dialog_dto(existing)
|
|
session.add(
|
|
IdempotencyRecord(
|
|
scope=scope,
|
|
user_id=user.id,
|
|
idempotency_key=idempotency_key,
|
|
request_fingerprint=fingerprint,
|
|
status="completed",
|
|
response_status=200,
|
|
response_body_json=json.loads(json.dumps(result, default=str)),
|
|
resource_type="dialog",
|
|
resource_id=existing.id,
|
|
expires_at=datetime.now(UTC) + timedelta(hours=24),
|
|
)
|
|
)
|
|
await session.commit()
|
|
return result, 200
|
|
dialog = Dialog(id=uuid.uuid4(), user_id=user.id, status="open")
|
|
session.add(dialog)
|
|
session.add(
|
|
audit(
|
|
"dialog.created",
|
|
context,
|
|
user.id,
|
|
"dialog",
|
|
dialog.id,
|
|
metadata={"status": dialog.status},
|
|
)
|
|
)
|
|
result = dialog_dto(dialog)
|
|
session.add(
|
|
IdempotencyRecord(
|
|
scope=scope,
|
|
user_id=user.id,
|
|
idempotency_key=idempotency_key,
|
|
request_fingerprint=fingerprint,
|
|
status="completed",
|
|
response_status=201,
|
|
response_body_json=json.loads(json.dumps(result, default=str)),
|
|
resource_type="dialog",
|
|
resource_id=dialog.id,
|
|
expires_at=datetime.now(UTC) + timedelta(hours=24),
|
|
)
|
|
)
|
|
await session.commit()
|
|
await session.refresh(dialog)
|
|
return dialog_dto(dialog), 201
|
|
|
|
|
|
async def init_attachment(
|
|
session: AsyncSession,
|
|
user: UserIdentity,
|
|
dialog_id: uuid.UUID,
|
|
body: AttachmentInitRequest,
|
|
snapshot: SettingsSnapshot,
|
|
s3: S3Client,
|
|
context: AuditContext,
|
|
) -> dict[str, Any]:
|
|
await owned_dialog(session, user.id, dialog_id)
|
|
extension = PurePath(body.file_name).suffix.lower().lstrip(".")
|
|
if (
|
|
extension not in snapshot.strings("chat.attachments.allowed_extensions")
|
|
or body.mime_type not in snapshot.strings("chat.attachments.allowed_mime_types")
|
|
or body.size_bytes > snapshot.integer("chat.attachments.max_size_mb") * 1024 * 1024
|
|
):
|
|
raise DomainError("validation_error", 400, "File type or size is not allowed")
|
|
attachment_id = uuid.uuid4()
|
|
key = f"quarantine/users/{user.id}/dialogs/{dialog_id}/{attachment_id}"
|
|
ttl = snapshot.integer("chat.attachments.presigned_upload_ttl_seconds")
|
|
expires = datetime.now(UTC) + timedelta(seconds=ttl)
|
|
safe_name = re.sub(r"[^A-Za-z0-9._-]", "_", unicodedata.normalize("NFKC", body.file_name))
|
|
item = MessageAttachment(
|
|
id=attachment_id,
|
|
dialog_id=dialog_id,
|
|
owner_user_id=user.id,
|
|
direction="client_upload",
|
|
original_file_name=body.file_name,
|
|
safe_file_name=safe_name,
|
|
mime_type=body.mime_type,
|
|
size_bytes=body.size_bytes,
|
|
scan_status="pending",
|
|
storage_bucket=s3.settings.selectel_s3_bucket_quarantine,
|
|
object_key=key,
|
|
quarantine_object_key=key,
|
|
upload_expires_at=expires,
|
|
)
|
|
session.add(item)
|
|
session.add(
|
|
audit(
|
|
"attachment.upload_initialized",
|
|
context,
|
|
user.id,
|
|
"attachment",
|
|
attachment_id,
|
|
metadata={"mime_type": body.mime_type, "size_bytes": body.size_bytes},
|
|
)
|
|
)
|
|
await session.commit()
|
|
url = await s3.presign_put(key, body.mime_type, ttl)
|
|
return {
|
|
"attachment_id": attachment_id,
|
|
"upload_url": url,
|
|
"upload_headers": {"Content-Type": body.mime_type},
|
|
"expires_at": expires,
|
|
}
|
|
|
|
|
|
async def complete_attachment(
|
|
session: AsyncSession,
|
|
user: UserIdentity,
|
|
dialog_id: uuid.UUID,
|
|
attachment_id: uuid.UUID,
|
|
body: AttachmentCompleteRequest,
|
|
s3: S3Client,
|
|
context: AuditContext,
|
|
) -> dict[str, Any]:
|
|
item = (
|
|
await session.execute(
|
|
select(MessageAttachment).where(
|
|
MessageAttachment.id == attachment_id,
|
|
MessageAttachment.dialog_id == dialog_id,
|
|
MessageAttachment.owner_user_id == user.id,
|
|
MessageAttachment.record_status == "A",
|
|
)
|
|
)
|
|
).scalar_one_or_none()
|
|
if item is None:
|
|
raise DomainError("not_found", 404, "Resource was not found")
|
|
checksum = body.checksum.removeprefix("sha256:")
|
|
if item.completed_at:
|
|
if item.checksum_sha256 != checksum:
|
|
raise DomainError("resource_state_conflict", 409, "Attachment checksum changed")
|
|
return attachment_dto(item)
|
|
try:
|
|
head = await s3.head(item.storage_bucket, item.object_key)
|
|
except Exception as exc:
|
|
raise DomainError("dependency_unavailable", 503, "Object storage is unavailable") from exc
|
|
if int(head["ContentLength"]) != item.size_bytes or head.get("ContentType") != item.mime_type:
|
|
raise DomainError("attachment_checksum_mismatch", 400, "Uploaded metadata does not match")
|
|
version_id, etag = head.get("VersionId"), head.get("ETag")
|
|
if not version_id or not etag:
|
|
raise DomainError(
|
|
"dependency_unavailable", 503, "Versioned object metadata is unavailable"
|
|
)
|
|
item.checksum_sha256 = checksum
|
|
item.quarantine_version_id = str(version_id)
|
|
item.quarantine_etag = str(etag)
|
|
item.completed_at = datetime.now(UTC)
|
|
session.add(
|
|
audit(
|
|
"attachment.upload_completed",
|
|
context,
|
|
user.id,
|
|
"attachment",
|
|
item.id,
|
|
metadata={"mime_type": item.mime_type, "size_bytes": item.size_bytes},
|
|
)
|
|
)
|
|
await session.commit()
|
|
return attachment_dto(item)
|
|
|
|
|
|
def attachment_dto(item: MessageAttachment) -> dict[str, Any]:
|
|
return {
|
|
"attachment_id": item.id,
|
|
"file_name": item.safe_file_name,
|
|
"mime_type": item.mime_type,
|
|
"size_bytes": item.size_bytes,
|
|
"checksum": f"sha256:{item.checksum_sha256}" if item.checksum_sha256 else None,
|
|
"scan_status": item.scan_status,
|
|
"completed_at": item.completed_at,
|
|
}
|
|
|
|
|
|
async def send_message(
|
|
session: AsyncSession,
|
|
user: UserIdentity,
|
|
dialog_id: uuid.UUID,
|
|
body: MessageRequest,
|
|
idem_key: str,
|
|
context: AuditContext,
|
|
business: SettingsSnapshot,
|
|
settings: Settings,
|
|
safety: SafetyClient,
|
|
openlines: OpenLinesClient,
|
|
s3: S3Client,
|
|
fanout: RealtimeFanout,
|
|
) -> dict[str, Any]:
|
|
scope = f"dialogs.{dialog_id}.messages.create"
|
|
fingerprint = hashlib.sha256(
|
|
json.dumps(body.model_dump(mode="json"), sort_keys=True, separators=(",", ":")).encode()
|
|
).hexdigest()
|
|
idem = (
|
|
await session.execute(
|
|
select(IdempotencyRecord).where(
|
|
IdempotencyRecord.scope == scope,
|
|
IdempotencyRecord.user_id == user.id,
|
|
IdempotencyRecord.idempotency_key == idem_key,
|
|
)
|
|
)
|
|
).scalar_one_or_none()
|
|
if idem:
|
|
if idem.request_fingerprint != fingerprint:
|
|
raise DomainError("idempotency_key_reused", 409, "Idempotency key was reused")
|
|
if idem.status == "completed" and idem.response_body_json:
|
|
if idem.response_status == 422:
|
|
raise DomainError("message_blocked", 422, "Message was blocked by safety policy")
|
|
return idem.response_body_json
|
|
prior = (
|
|
await session.execute(
|
|
select(Message).where(
|
|
Message.dialog_id == dialog_id,
|
|
Message.client_idempotency_key == idem_key,
|
|
)
|
|
)
|
|
).scalar_one_or_none()
|
|
if prior and prior.delivery_status == "delivered":
|
|
return message_dto(prior)
|
|
raise DomainError(
|
|
"dependency_unavailable",
|
|
503,
|
|
"Previous request is still being recovered",
|
|
{"retry_after": 2},
|
|
)
|
|
dialog = await owned_dialog(session, user.id, dialog_id)
|
|
now, message_id = datetime.now(UTC), uuid.uuid4()
|
|
idem = IdempotencyRecord(
|
|
scope=scope,
|
|
user_id=user.id,
|
|
idempotency_key=idem_key,
|
|
request_fingerprint=fingerprint,
|
|
status="in_progress",
|
|
resource_type="message",
|
|
resource_id=message_id,
|
|
expires_at=now + timedelta(hours=24),
|
|
)
|
|
session.add(idem)
|
|
attachment: MessageAttachment | None = None
|
|
if isinstance(body, FileMessageRequest):
|
|
attachment = (
|
|
await session.execute(
|
|
select(MessageAttachment).where(
|
|
MessageAttachment.id == body.attachment_id,
|
|
MessageAttachment.owner_user_id == user.id,
|
|
MessageAttachment.dialog_id == dialog_id,
|
|
MessageAttachment.record_status == "A",
|
|
)
|
|
)
|
|
).scalar_one_or_none()
|
|
if not attachment or not attachment.completed_at:
|
|
raise DomainError("attachment_not_completed", 400, "Attachment upload is incomplete")
|
|
if attachment.checksum_sha256 != body.checksum.removeprefix("sha256:"):
|
|
raise DomainError("attachment_checksum_mismatch", 400, "Attachment checksum differs")
|
|
text, kind = "", "file"
|
|
else:
|
|
text, kind = unicodedata.normalize("NFKC", body.text).strip(), "text"
|
|
max_length = business.integer(CHAT_MESSAGE_MAX_LENGTH_KEY)
|
|
if len(text) > max_length:
|
|
raise DomainError(
|
|
"message_too_long",
|
|
422,
|
|
"Message exceeds the configured maximum length",
|
|
{"max_length": max_length},
|
|
)
|
|
message = Message(
|
|
id=message_id,
|
|
dialog_id=dialog_id,
|
|
sender_type="client",
|
|
content_kind=kind,
|
|
text=text,
|
|
safety_status="pending",
|
|
delivery_status="accepted",
|
|
client_idempotency_key=idem_key,
|
|
occurred_at=now,
|
|
)
|
|
session.add(message)
|
|
if attachment:
|
|
attachment.message_id = message_id
|
|
session.add(
|
|
audit(
|
|
"message.submitted",
|
|
context,
|
|
user.id,
|
|
"message",
|
|
message_id,
|
|
metadata={"content_kind": body.content_kind, "delivery_status": "processing"},
|
|
)
|
|
)
|
|
await session.commit()
|
|
await publish_message(fanout, message, settings, [attachment] if attachment else [])
|
|
payload: dict[str, Any] = {"message_id": str(message_id), "content_kind": kind, "text": text}
|
|
if attachment:
|
|
payload["attachment"] = {
|
|
"attachment_id": str(attachment.id),
|
|
"quarantine_object_key": attachment.quarantine_object_key,
|
|
"quarantine_version_id": attachment.quarantine_version_id,
|
|
"quarantine_etag": attachment.quarantine_etag,
|
|
"checksum": f"sha256:{attachment.checksum_sha256}",
|
|
"mime_type": attachment.mime_type,
|
|
"size_bytes": attachment.size_bytes,
|
|
}
|
|
outbox: DeliveryOutbox | None = None
|
|
try:
|
|
verdict = await safety.check(payload, context.request_id)
|
|
task: SafetyTask | None = None
|
|
if verdict["_status"] == 202:
|
|
task_id = verdict["task_id"]
|
|
task = SafetyTask(
|
|
task_id=task_id,
|
|
poll_location=verdict["_location"],
|
|
message_id=message.id,
|
|
attachment_id=attachment.id if attachment else None,
|
|
quarantine_object_key=attachment.quarantine_object_key if attachment else None,
|
|
status="polling",
|
|
processing_mode=verdict["processing_mode"],
|
|
config_version=verdict["config_version"],
|
|
rules_version=verdict["rules_version"],
|
|
expires_at=datetime.fromisoformat(verdict["expires_at"].replace("Z", "+00:00")),
|
|
deadline_at=now
|
|
+ timedelta(seconds=settings.message_safety_task_poll_max_sec + 900),
|
|
next_poll_at=now,
|
|
)
|
|
session.add(task)
|
|
await session.commit()
|
|
deadline = time_monotonic() + settings.message_safety_task_poll_max_sec
|
|
while time_monotonic() < deadline:
|
|
await sleep(settings.message_safety_task_poll_interval_sec)
|
|
verdict = await safety.poll(task.poll_location, context.request_id)
|
|
if verdict["_status"] != 202:
|
|
break
|
|
task.poll_location = verdict["_location"]
|
|
else:
|
|
raise DependencyFailure(timeout=True)
|
|
message.safety_processing_mode = verdict["processing_mode"]
|
|
message.safety_config_version = verdict["config_version"]
|
|
message.safety_rules_version = verdict["rules_version"]
|
|
if task:
|
|
task.status = "completed"
|
|
if verdict["_status"] == 403:
|
|
message.text = ""
|
|
message.safety_status = "blocked"
|
|
message.delivery_status = "rejected"
|
|
reply = safety_reply_message(dialog_id, kind)
|
|
session.add(reply)
|
|
dialog.last_message_at = reply.occurred_at
|
|
if attachment and attachment.quarantine_object_key:
|
|
attachment.scan_status = "infected"
|
|
await s3.delete_quarantine(attachment.quarantine_object_key)
|
|
session.add(
|
|
audit(
|
|
"message.blocked",
|
|
context,
|
|
user.id,
|
|
"message",
|
|
message.id,
|
|
metadata={
|
|
"content_kind": message.content_kind,
|
|
"safety_status": message.safety_status,
|
|
"delivery_status": message.delivery_status,
|
|
},
|
|
)
|
|
)
|
|
idem.status = "completed"
|
|
idem.response_status = 422
|
|
idem.response_body_json = {
|
|
"error": {"code": "message_blocked", "message_id": str(message.id)}
|
|
}
|
|
await session.commit()
|
|
await publish_message_status(fanout, message, settings)
|
|
await publish_message(fanout, reply, settings)
|
|
await publish_dialog_status(fanout, dialog)
|
|
raise DomainError("message_blocked", 422, "Message was blocked by safety policy")
|
|
if verdict["_status"] != 200:
|
|
raise DependencyFailure()
|
|
if attachment and attachment.quarantine_object_key:
|
|
destination = f"attachments/dialogs/{dialog_id}/{attachment.id}"
|
|
await s3.promote(
|
|
attachment.quarantine_object_key,
|
|
destination,
|
|
version_id=attachment.quarantine_version_id or "",
|
|
etag=attachment.quarantine_etag or "",
|
|
)
|
|
attachment.storage_bucket = settings.selectel_s3_bucket_attachments
|
|
attachment.object_key = destination
|
|
attachment.quarantine_object_key = None
|
|
attachment.scan_status = (
|
|
"bypassed" if verdict["processing_mode"] == "mock" else "clean"
|
|
)
|
|
message.safety_status = "allowed"
|
|
outbox = DeliveryOutbox(
|
|
message_id=message.id,
|
|
external_chat_id=dialog_id,
|
|
payload_json={
|
|
"message_id": str(message.id),
|
|
"external_chat_id": str(dialog_id),
|
|
"occurred_at": message.occurred_at.isoformat(),
|
|
"user": {"id": str(user.id), "display_name": user.phone_number},
|
|
"message": {
|
|
"content_kind": message.content_kind,
|
|
"text": message.text,
|
|
"files": (
|
|
[
|
|
{
|
|
"attachment_id": str(attachment.id),
|
|
"name": attachment.safe_file_name,
|
|
"mime_type": attachment.mime_type,
|
|
"size_bytes": attachment.size_bytes,
|
|
"_storage_bucket": attachment.storage_bucket,
|
|
"_object_key": attachment.object_key,
|
|
}
|
|
]
|
|
if attachment
|
|
else []
|
|
),
|
|
},
|
|
},
|
|
# Keep the row recoverable after a process crash, but do not let the
|
|
# delivery worker race the synchronous first attempt.
|
|
next_attempt_at=datetime.now(UTC)
|
|
+ timedelta(seconds=settings.bitrix_local_app_http_timeout_sec + 5),
|
|
)
|
|
session.add(outbox)
|
|
await session.commit()
|
|
await publish_message_status(fanout, message, settings)
|
|
delivery_payload = await fresh_openlines_payload(outbox.payload_json, s3)
|
|
await openlines.send(message.id, delivery_payload, context.request_id)
|
|
message.delivery_status = "delivered"
|
|
dialog.status = "waiting_for_company"
|
|
dialog.last_message_at = datetime.now(UTC)
|
|
outbox.status = "delivered"
|
|
session.add(
|
|
audit(
|
|
"message.delivered",
|
|
context,
|
|
user.id,
|
|
"message",
|
|
message.id,
|
|
metadata={
|
|
"content_kind": message.content_kind,
|
|
"safety_status": message.safety_status,
|
|
"delivery_status": message.delivery_status,
|
|
},
|
|
outcome="blocked",
|
|
)
|
|
)
|
|
result = message_dto(message, [attachment] if attachment else [])
|
|
idem.status = "completed"
|
|
idem.response_status = 201
|
|
idem.response_body_json = json.loads(json.dumps(result, default=str))
|
|
await session.commit()
|
|
await publish_message_status(fanout, message, settings)
|
|
await publish_dialog_status(fanout, dialog)
|
|
return result
|
|
except DomainError:
|
|
raise
|
|
except DependencyFailure as exc:
|
|
message.delivery_status = "failed"
|
|
if outbox is not None:
|
|
outbox.attempt_count += 1
|
|
outbox.status = "retry"
|
|
outbox.next_attempt_at = datetime.now(UTC) + timedelta(
|
|
seconds=min(3600, 2**outbox.attempt_count)
|
|
)
|
|
outbox.last_error_code = (
|
|
"dependency_timeout" if exc.timeout else "dependency_unavailable"
|
|
)
|
|
outbox.locked_at = None
|
|
outbox.locked_by = None
|
|
session.add(
|
|
audit(
|
|
"message.failed",
|
|
context,
|
|
user.id,
|
|
"message",
|
|
message.id,
|
|
metadata={
|
|
"content_kind": message.content_kind,
|
|
"delivery_status": message.delivery_status,
|
|
},
|
|
outcome="failed",
|
|
)
|
|
)
|
|
await session.commit()
|
|
await publish_message_status(fanout, message, settings)
|
|
raise DomainError(
|
|
"dependency_timeout" if exc.timeout else "dependency_unavailable",
|
|
504 if exc.timeout else 503,
|
|
"A required dependency did not complete the request",
|
|
) from exc
|
|
|
|
|
|
async def apply_inbox(
|
|
session: AsyncSession,
|
|
event: OpenLinesInbox,
|
|
context: AuditContext,
|
|
snapshot: SettingsSnapshot,
|
|
s3: S3Client,
|
|
http: httpx.AsyncClient,
|
|
settings: Settings,
|
|
fanout: RealtimeFanout,
|
|
) -> tuple[dict[str, Any], int]:
|
|
fingerprint = hashlib.sha256(event.model_dump_json().encode()).hexdigest()
|
|
existing = (
|
|
await session.execute(
|
|
select(OpenLinesInboxReceipt).where(OpenLinesInboxReceipt.event_id == event.event_id)
|
|
)
|
|
).scalar_one_or_none()
|
|
if existing:
|
|
if existing.payload_fingerprint != fingerprint:
|
|
raise DomainError("idempotency_key_reused", 409, "Event id was reused")
|
|
return {"status": "duplicate"}, 200
|
|
dialog = (
|
|
await session.execute(
|
|
select(Dialog).where(Dialog.id == event.external_chat_id, Dialog.record_status == "A")
|
|
)
|
|
).scalar_one_or_none()
|
|
if not dialog:
|
|
raise DomainError("not_found", 404, "Resource was not found")
|
|
receipt = OpenLinesInboxReceipt(
|
|
event_id=event.event_id,
|
|
external_chat_id=event.external_chat_id,
|
|
bitrix_message_id=event.bitrix_message_id,
|
|
event_type=event.event_type,
|
|
payload_fingerprint=fingerprint,
|
|
status="processing",
|
|
)
|
|
session.add(receipt)
|
|
message: Message | None = None
|
|
attachment: MessageAttachment | None = None
|
|
if event.event_type == "dialog.closed":
|
|
dialog.status = "closed"
|
|
dialog.closed_at = event.occurred_at
|
|
else:
|
|
assert event.message is not None
|
|
inbound_file = event.message.files[0] if event.message.files else None
|
|
attachment_data: tuple[str, int, str] | None = None
|
|
if inbound_file:
|
|
extension = PurePath(inbound_file.name).suffix.lower().lstrip(".")
|
|
max_bytes = snapshot.integer("chat.attachments.max_size_mb") * 1024 * 1024
|
|
if (
|
|
extension not in snapshot.strings("chat.attachments.allowed_extensions")
|
|
or inbound_file.mime_type
|
|
not in snapshot.strings("chat.attachments.allowed_mime_types")
|
|
or inbound_file.size_bytes > max_bytes
|
|
):
|
|
raise DomainError("validation_error", 400, "Inbound file is not allowed")
|
|
attachment_id = uuid.uuid4()
|
|
object_key = f"attachments/dialogs/{dialog.id}/{attachment_id}"
|
|
try:
|
|
actual_size, checksum = await s3.upload_inbound(
|
|
http,
|
|
str(inbound_file.download_url),
|
|
object_key,
|
|
inbound_file.mime_type,
|
|
max_bytes,
|
|
)
|
|
except DependencyFailure as exc:
|
|
raise DomainError(
|
|
"dependency_unavailable", 503, "Inbound file transfer failed"
|
|
) from exc
|
|
if actual_size != inbound_file.size_bytes:
|
|
raise DomainError("validation_error", 400, "Inbound file size differs")
|
|
attachment_data = object_key, actual_size, checksum
|
|
message = Message(
|
|
dialog_id=dialog.id,
|
|
sender_type="company",
|
|
content_kind="file" if inbound_file else "text",
|
|
text=event.message.text,
|
|
safety_status="allowed",
|
|
delivery_status="delivered",
|
|
external_message_id=event.bitrix_message_id,
|
|
occurred_at=event.occurred_at,
|
|
)
|
|
session.add(message)
|
|
await session.flush()
|
|
if inbound_file and attachment_data:
|
|
object_key, actual_size, checksum = attachment_data
|
|
safe_name = re.sub(
|
|
r"[^A-Za-z0-9._-]",
|
|
"_",
|
|
unicodedata.normalize("NFKC", inbound_file.name),
|
|
)
|
|
attachment = MessageAttachment(
|
|
dialog_id=dialog.id,
|
|
message_id=message.id,
|
|
owner_user_id=dialog.user_id,
|
|
direction="company_inbound",
|
|
original_file_name=inbound_file.name,
|
|
safe_file_name=safe_name,
|
|
mime_type=inbound_file.mime_type,
|
|
size_bytes=actual_size,
|
|
checksum_sha256=checksum,
|
|
scan_status="clean",
|
|
storage_bucket=s3.settings.selectel_s3_bucket_attachments,
|
|
object_key=object_key,
|
|
completed_at=datetime.now(UTC),
|
|
)
|
|
session.add(attachment)
|
|
receipt.message_id = message.id
|
|
dialog.status = "waiting_for_client"
|
|
dialog.last_message_at = event.occurred_at
|
|
receipt.status = "applied"
|
|
session.add(
|
|
audit(
|
|
"openlines.inbox_applied",
|
|
context,
|
|
None,
|
|
"dialog",
|
|
dialog.id,
|
|
metadata={"event_type": event.event_type},
|
|
)
|
|
)
|
|
await session.commit()
|
|
if message is not None:
|
|
await publish_message(fanout, message, settings, [attachment] if attachment else [])
|
|
await publish_dialog_status(fanout, dialog)
|
|
return {"status": "applied"}, 201
|
|
|
|
|
|
async def sleep(seconds: float) -> None:
|
|
import asyncio
|
|
|
|
await asyncio.sleep(seconds)
|
|
|
|
|
|
def time_monotonic() -> float:
|
|
import time
|
|
|
|
return time.monotonic()
|