Разработана первая версия приложений
This commit is contained in:
mi
@@ -0,0 +1 @@
|
||||
"""HAN Chat API backend."""
|
||||
@@ -0,0 +1,106 @@
|
||||
import asyncio
|
||||
import time
|
||||
from dataclasses import dataclass
|
||||
from typing import Any
|
||||
|
||||
import httpx
|
||||
import jwt
|
||||
import phonenumbers
|
||||
from jwt import ExpiredSignatureError, InvalidTokenError, PyJWK
|
||||
|
||||
from app.settings import Settings
|
||||
|
||||
|
||||
class AuthError(Exception):
|
||||
def __init__(self, code: str = "unauthorized") -> None:
|
||||
self.code = code
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class Principal:
|
||||
subject: str
|
||||
phone_number: str | None
|
||||
claims: dict[str, Any]
|
||||
|
||||
|
||||
class JWKSValidator:
|
||||
def __init__(self, settings: Settings, http: httpx.AsyncClient) -> None:
|
||||
self.settings = settings
|
||||
self.http = http
|
||||
self._keys: dict[str, PyJWK] = {}
|
||||
self._loaded_at = 0.0
|
||||
self._lock = asyncio.Lock()
|
||||
|
||||
async def refresh(self) -> None:
|
||||
async with self._lock:
|
||||
discovery_url = (
|
||||
f"{str(self.settings.keycloak_internal_url).rstrip('/')}"
|
||||
f"/realms/{self.settings.keycloak_realm}/.well-known/openid-configuration"
|
||||
)
|
||||
discovery = (await self.http.get(discovery_url, timeout=3)).raise_for_status().json()
|
||||
jwks_uri = discovery["jwks_uri"]
|
||||
public_prefix = str(self.settings.keycloak_public_url).rstrip("/")
|
||||
internal_prefix = str(self.settings.keycloak_internal_url).rstrip("/")
|
||||
if jwks_uri.startswith(public_prefix):
|
||||
jwks_uri = internal_prefix + jwks_uri[len(public_prefix) :]
|
||||
payload = (await self.http.get(jwks_uri, timeout=3)).raise_for_status().json()
|
||||
self._keys = {
|
||||
key["kid"]: PyJWK.from_dict(key)
|
||||
for key in payload.get("keys", [])
|
||||
if key.get("kid") and key.get("kty") == "RSA"
|
||||
}
|
||||
self._loaded_at = time.monotonic()
|
||||
|
||||
async def validate(self, token: str) -> Principal:
|
||||
try:
|
||||
header = jwt.get_unverified_header(token)
|
||||
except InvalidTokenError as exc:
|
||||
raise AuthError() from exc
|
||||
if header.get("alg") != "RS256" or not header.get("kid"):
|
||||
raise AuthError()
|
||||
kid = str(header["kid"])
|
||||
stale = time.monotonic() - self._loaded_at > self.settings.jwks_cache_ttl_seconds
|
||||
if stale or kid not in self._keys:
|
||||
try:
|
||||
await self.refresh()
|
||||
except (httpx.HTTPError, KeyError, ValueError):
|
||||
if (
|
||||
kid not in self._keys
|
||||
or time.monotonic() - self._loaded_at > self.settings.jwks_stale_grace_seconds
|
||||
):
|
||||
raise AuthError() from None
|
||||
key = self._keys.get(kid)
|
||||
if key is None:
|
||||
raise AuthError()
|
||||
try:
|
||||
claims = jwt.decode(
|
||||
token,
|
||||
key.key,
|
||||
algorithms=["RS256"],
|
||||
audience=self.settings.keycloak_audience,
|
||||
issuer=self.settings.issuer,
|
||||
options={"require": ["exp", "sub"]},
|
||||
leeway=30,
|
||||
)
|
||||
except ExpiredSignatureError as exc:
|
||||
raise AuthError("token_expired") from exc
|
||||
except InvalidTokenError as exc:
|
||||
raise AuthError() from exc
|
||||
subject = claims.get("sub")
|
||||
if not isinstance(subject, str) or not subject:
|
||||
raise AuthError()
|
||||
return Principal(subject, canonical_phone(claims), claims)
|
||||
|
||||
|
||||
def canonical_phone(claims: dict[str, Any]) -> str | None:
|
||||
for name in ("phone_number", "preferred_username"):
|
||||
value = claims.get(name)
|
||||
if not isinstance(value, str):
|
||||
continue
|
||||
try:
|
||||
number = phonenumbers.parse(value, None)
|
||||
except phonenumbers.NumberParseException:
|
||||
continue
|
||||
if phonenumbers.is_valid_number(number) and value.startswith("+"):
|
||||
return phonenumbers.format_number(number, phonenumbers.PhoneNumberFormat.E164)
|
||||
return None
|
||||
@@ -0,0 +1 @@
|
||||
"""Operational command-line entry points."""
|
||||
@@ -0,0 +1,115 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import asyncio
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import yaml
|
||||
from sqlalchemy import func, or_
|
||||
from sqlalchemy.dialects.postgresql import insert
|
||||
|
||||
from app.db import AppSetting, Database
|
||||
from app.settings import get_settings
|
||||
|
||||
VALUE_TYPES = {"boolean", "integer", "string", "string_list"}
|
||||
|
||||
|
||||
def load_seed(path: Path) -> list[dict[str, Any]]:
|
||||
document = yaml.safe_load(path.read_text(encoding="utf-8"))
|
||||
if not isinstance(document, dict) or document.get("schema_version") != 1:
|
||||
raise ValueError("settings file must have schema_version: 1")
|
||||
settings = document.get("settings")
|
||||
if not isinstance(settings, dict) or not settings:
|
||||
raise ValueError("settings file must contain a non-empty settings mapping")
|
||||
|
||||
rows: list[dict[str, Any]] = []
|
||||
for key, raw in settings.items():
|
||||
if not isinstance(key, str) or not key.strip():
|
||||
raise ValueError("setting keys must be non-empty strings")
|
||||
if not isinstance(raw, dict):
|
||||
raise ValueError(f"{key}: setting must be a mapping")
|
||||
value_type = raw.get("type")
|
||||
if value_type not in VALUE_TYPES:
|
||||
raise ValueError(f"{key}: unsupported type {value_type!r}")
|
||||
if not isinstance(raw.get("public"), bool):
|
||||
raise ValueError(f"{key}: public must be a boolean")
|
||||
description = raw.get("description")
|
||||
if description is not None and not isinstance(description, str):
|
||||
raise ValueError(f"{key}: description must be a string")
|
||||
rows.append(
|
||||
{
|
||||
"setting_key": key,
|
||||
"setting_value": serialize_value(key, value_type, raw.get("value")),
|
||||
"value_type": value_type,
|
||||
"is_public": raw["public"],
|
||||
"description": description,
|
||||
"record_status": "A",
|
||||
}
|
||||
)
|
||||
return rows
|
||||
|
||||
|
||||
def serialize_value(key: str, value_type: str, value: Any) -> str:
|
||||
if value_type == "boolean":
|
||||
if not isinstance(value, bool):
|
||||
raise ValueError(f"{key}: boolean value expected")
|
||||
return str(value).lower()
|
||||
if value_type == "integer":
|
||||
if not isinstance(value, int) or isinstance(value, bool):
|
||||
raise ValueError(f"{key}: integer value expected")
|
||||
return str(value)
|
||||
if value_type == "string_list":
|
||||
if isinstance(value, list) and all(isinstance(item, str) for item in value):
|
||||
return ",".join(value)
|
||||
if isinstance(value, str):
|
||||
return value
|
||||
raise ValueError(f"{key}: string or list of strings expected")
|
||||
if not isinstance(value, str):
|
||||
raise ValueError(f"{key}: string value expected")
|
||||
return value
|
||||
|
||||
|
||||
async def seed(path: Path) -> int:
|
||||
rows = load_seed(path)
|
||||
database = Database(get_settings().database_url)
|
||||
try:
|
||||
async with database.sessions() as session:
|
||||
for row in rows:
|
||||
statement = insert(AppSetting).values(**row)
|
||||
excluded = statement.excluded
|
||||
statement = statement.on_conflict_do_update(
|
||||
index_elements=[AppSetting.setting_key],
|
||||
set_={
|
||||
"setting_value": excluded.setting_value,
|
||||
"value_type": excluded.value_type,
|
||||
"is_public": excluded.is_public,
|
||||
"description": excluded.description,
|
||||
"record_status": "A",
|
||||
"updated_at": func.now(),
|
||||
},
|
||||
where=or_(
|
||||
AppSetting.setting_value.is_distinct_from(excluded.setting_value),
|
||||
AppSetting.value_type.is_distinct_from(excluded.value_type),
|
||||
AppSetting.is_public.is_distinct_from(excluded.is_public),
|
||||
AppSetting.description.is_distinct_from(excluded.description),
|
||||
AppSetting.record_status != "A",
|
||||
),
|
||||
)
|
||||
await session.execute(statement)
|
||||
await session.commit()
|
||||
finally:
|
||||
await database.close()
|
||||
return len(rows)
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(description="Idempotently seed application settings")
|
||||
parser.add_argument("--file", required=True, type=Path)
|
||||
args = parser.parse_args()
|
||||
count = asyncio.run(seed(args.file))
|
||||
print(f"Application settings seeded: {count}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,40 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
|
||||
from sqlalchemy import select
|
||||
|
||||
from app.db import AppSetting, Database
|
||||
from app.services import REQUIRED_SETTINGS
|
||||
from app.settings import get_settings
|
||||
|
||||
|
||||
async def validate() -> int:
|
||||
database = Database(get_settings().database_url)
|
||||
try:
|
||||
async with database.sessions() as session:
|
||||
active_keys = set(
|
||||
(
|
||||
await session.execute(
|
||||
select(AppSetting.setting_key).where(AppSetting.record_status == "A")
|
||||
)
|
||||
)
|
||||
.scalars()
|
||||
.all()
|
||||
)
|
||||
finally:
|
||||
await database.close()
|
||||
|
||||
missing = sorted(REQUIRED_SETTINGS - active_keys)
|
||||
if missing:
|
||||
raise RuntimeError(f"Mandatory application settings are missing: {', '.join(missing)}")
|
||||
return len(REQUIRED_SETTINGS)
|
||||
|
||||
|
||||
def main() -> None:
|
||||
count = asyncio.run(validate())
|
||||
print(f"Mandatory application settings validated: {count}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,358 @@
|
||||
import uuid
|
||||
from collections.abc import AsyncIterator
|
||||
from datetime import datetime
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import (
|
||||
JSON,
|
||||
BigInteger,
|
||||
Boolean,
|
||||
CheckConstraint,
|
||||
DateTime,
|
||||
ForeignKey,
|
||||
Index,
|
||||
Integer,
|
||||
String,
|
||||
Text,
|
||||
UniqueConstraint,
|
||||
func,
|
||||
)
|
||||
from sqlalchemy.dialects.postgresql import INET, UUID
|
||||
from sqlalchemy.ext.asyncio import (
|
||||
AsyncEngine,
|
||||
AsyncSession,
|
||||
async_sessionmaker,
|
||||
create_async_engine,
|
||||
)
|
||||
from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column
|
||||
|
||||
SCHEMA = "han_app"
|
||||
|
||||
|
||||
class Base(DeclarativeBase):
|
||||
type_annotation_map = {dict[str, Any]: JSON}
|
||||
|
||||
|
||||
class Common:
|
||||
id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
|
||||
record_status: Mapped[str] = mapped_column(String(1), default="A", server_default="A")
|
||||
status_changed_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||
status_change_reason: Mapped[str | None] = mapped_column(String(255))
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
|
||||
updated_at: Mapped[datetime] = mapped_column(
|
||||
DateTime(timezone=True), server_default=func.now(), onupdate=func.now()
|
||||
)
|
||||
updater_user_id: Mapped[uuid.UUID | None] = mapped_column(UUID(as_uuid=True))
|
||||
|
||||
|
||||
class UserIdentity(Common, Base):
|
||||
__tablename__ = "user_identities"
|
||||
__table_args__ = (Index("ix_user_identities_phone", "phone_number"), {"schema": SCHEMA})
|
||||
keycloak_sub: Mapped[str] = mapped_column(String(255), unique=True)
|
||||
phone_number: Mapped[str] = mapped_column(String(32))
|
||||
last_login_at: Mapped[datetime] = mapped_column(DateTime(timezone=True))
|
||||
|
||||
|
||||
class ClientProfile(Common, Base):
|
||||
__tablename__ = "client_profiles"
|
||||
__table_args__ = ({"schema": SCHEMA},)
|
||||
user_id: Mapped[uuid.UUID] = mapped_column(
|
||||
ForeignKey(f"{SCHEMA}.user_identities.id", ondelete="RESTRICT"), unique=True
|
||||
)
|
||||
bitrix_contact_id: Mapped[str | None] = mapped_column(String(64))
|
||||
full_name: Mapped[str | None] = mapped_column(String(255))
|
||||
citizenship: Mapped[str | None] = mapped_column(String(128))
|
||||
russian_phone: Mapped[str | None] = mapped_column(String(32))
|
||||
foreign_phone: Mapped[str | None] = mapped_column(String(32))
|
||||
email: Mapped[str | None] = mapped_column(String(320))
|
||||
source_updated_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||
|
||||
|
||||
class UserConsent(Common, Base):
|
||||
__tablename__ = "user_consents"
|
||||
__table_args__ = (
|
||||
UniqueConstraint("user_id", "consent_type", "document_version"),
|
||||
CheckConstraint("consent_type IN ('personal_data','user_agreement','marketing')"),
|
||||
{"schema": SCHEMA},
|
||||
)
|
||||
user_id: Mapped[uuid.UUID] = mapped_column(
|
||||
ForeignKey(f"{SCHEMA}.user_identities.id", ondelete="RESTRICT")
|
||||
)
|
||||
ux_session_id: Mapped[uuid.UUID | None] = mapped_column(UUID(as_uuid=True))
|
||||
consent_type: Mapped[str] = mapped_column(String(32))
|
||||
document_version: Mapped[str] = mapped_column(String(64))
|
||||
accepted: Mapped[bool] = mapped_column(Boolean)
|
||||
accepted_at: Mapped[datetime] = mapped_column(DateTime(timezone=True))
|
||||
client_ip: Mapped[str | None] = mapped_column(INET)
|
||||
user_agent_hash: Mapped[str | None] = mapped_column(String(64))
|
||||
|
||||
|
||||
class UxSession(Common, Base):
|
||||
__tablename__ = "ux_sessions"
|
||||
__table_args__ = (
|
||||
CheckConstraint("start_reason IN ('first_launch','cold_start','idle_timeout')"),
|
||||
Index("ix_ux_sessions_user_started", "user_id", "started_at"),
|
||||
{"schema": SCHEMA},
|
||||
)
|
||||
user_id: Mapped[uuid.UUID] = mapped_column(
|
||||
ForeignKey(f"{SCHEMA}.user_identities.id", ondelete="RESTRICT")
|
||||
)
|
||||
start_reason: Mapped[str] = mapped_column(String(32))
|
||||
platform: Mapped[str] = mapped_column(String(32))
|
||||
app_version: Mapped[str] = mapped_column(String(64))
|
||||
device_id_hash: Mapped[str | None] = mapped_column(String(64))
|
||||
started_at: Mapped[datetime] = mapped_column(DateTime(timezone=True))
|
||||
|
||||
|
||||
class Dialog(Common, Base):
|
||||
__tablename__ = "dialogs"
|
||||
__table_args__ = (
|
||||
CheckConstraint("status IN ('open','waiting_for_company','waiting_for_client','closed')"),
|
||||
Index("ix_dialogs_user_updated", "user_id", "updated_at", "id"),
|
||||
{"schema": SCHEMA},
|
||||
)
|
||||
user_id: Mapped[uuid.UUID] = mapped_column(
|
||||
ForeignKey(f"{SCHEMA}.user_identities.id", ondelete="RESTRICT")
|
||||
)
|
||||
status: Mapped[str] = mapped_column(String(32), default="open")
|
||||
last_message_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||
closed_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||
|
||||
|
||||
class Message(Common, Base):
|
||||
__tablename__ = "messages"
|
||||
__table_args__ = (
|
||||
CheckConstraint("sender_type IN ('client','company')"),
|
||||
CheckConstraint("content_kind IN ('text','file')"),
|
||||
CheckConstraint("safety_status IN ('pending','allowed','blocked','needs_review')"),
|
||||
CheckConstraint(
|
||||
"delivery_status IN ('accepted','processing','delivered','rejected','failed')"
|
||||
),
|
||||
Index("ix_messages_dialog_created", "dialog_id", "created_at", "id"),
|
||||
UniqueConstraint("dialog_id", "client_idempotency_key"),
|
||||
{"schema": SCHEMA},
|
||||
)
|
||||
dialog_id: Mapped[uuid.UUID] = mapped_column(
|
||||
ForeignKey(f"{SCHEMA}.dialogs.id", ondelete="RESTRICT")
|
||||
)
|
||||
sender_type: Mapped[str] = mapped_column(String(16))
|
||||
content_kind: Mapped[str] = mapped_column(String(16))
|
||||
text: Mapped[str] = mapped_column(Text, default="")
|
||||
safety_status: Mapped[str] = mapped_column(String(16))
|
||||
delivery_status: Mapped[str] = mapped_column(String(16))
|
||||
external_message_id: Mapped[str | None] = mapped_column(String(255))
|
||||
client_idempotency_key: Mapped[str | None] = mapped_column(String(128))
|
||||
occurred_at: Mapped[datetime] = mapped_column(DateTime(timezone=True))
|
||||
|
||||
|
||||
class MessageAttachment(Common, Base):
|
||||
__tablename__ = "message_attachments"
|
||||
__table_args__ = (
|
||||
CheckConstraint("direction IN ('client_upload','company_inbound')"),
|
||||
CheckConstraint("scan_status IN ('pending','clean','infected','failed')"),
|
||||
CheckConstraint("size_bytes > 0"),
|
||||
{"schema": SCHEMA},
|
||||
)
|
||||
dialog_id: Mapped[uuid.UUID] = mapped_column(ForeignKey(f"{SCHEMA}.dialogs.id"))
|
||||
message_id: Mapped[uuid.UUID | None] = mapped_column(
|
||||
ForeignKey(f"{SCHEMA}.messages.id"), unique=True
|
||||
)
|
||||
owner_user_id: Mapped[uuid.UUID] = mapped_column(ForeignKey(f"{SCHEMA}.user_identities.id"))
|
||||
direction: Mapped[str] = mapped_column(String(32))
|
||||
original_file_name: Mapped[str] = mapped_column(String(255))
|
||||
safe_file_name: Mapped[str] = mapped_column(String(255))
|
||||
mime_type: Mapped[str] = mapped_column(String(128))
|
||||
size_bytes: Mapped[int] = mapped_column(BigInteger)
|
||||
checksum_sha256: Mapped[str | None] = mapped_column(String(64))
|
||||
scan_status: Mapped[str] = mapped_column(String(16), default="pending")
|
||||
storage_bucket: Mapped[str] = mapped_column(String(255))
|
||||
object_key: Mapped[str] = mapped_column(String(1024))
|
||||
quarantine_object_key: Mapped[str | None] = mapped_column(String(1024))
|
||||
upload_expires_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||
completed_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||
|
||||
|
||||
class Document(Common, Base):
|
||||
__tablename__ = "documents"
|
||||
__table_args__ = (UniqueConstraint("storage_bucket", "object_key"), {"schema": SCHEMA})
|
||||
user_id: Mapped[uuid.UUID] = mapped_column(ForeignKey(f"{SCHEMA}.user_identities.id"))
|
||||
name: Mapped[str] = mapped_column(String(255))
|
||||
mime_type: Mapped[str] = mapped_column(String(128))
|
||||
size_bytes: Mapped[int] = mapped_column(BigInteger)
|
||||
checksum_sha256: Mapped[str] = mapped_column(String(64))
|
||||
storage_bucket: Mapped[str] = mapped_column(String(255))
|
||||
object_key: Mapped[str] = mapped_column(String(1024))
|
||||
sent_at: Mapped[datetime] = mapped_column(DateTime(timezone=True))
|
||||
|
||||
|
||||
class SafetyTask(Base):
|
||||
__tablename__ = "safety_tasks"
|
||||
__table_args__ = ({"schema": SCHEMA},)
|
||||
id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
|
||||
task_id: Mapped[str] = mapped_column(String(255), unique=True)
|
||||
message_id: Mapped[uuid.UUID] = mapped_column(ForeignKey(f"{SCHEMA}.messages.id"), unique=True)
|
||||
attachment_id: Mapped[uuid.UUID | None] = mapped_column(UUID(as_uuid=True))
|
||||
quarantine_object_key: Mapped[str | None] = mapped_column(String(1024))
|
||||
status: Mapped[str] = mapped_column(String(16))
|
||||
deadline_at: Mapped[datetime] = mapped_column(DateTime(timezone=True))
|
||||
next_poll_at: Mapped[datetime] = mapped_column(DateTime(timezone=True))
|
||||
attempt_count: Mapped[int] = mapped_column(Integer, default=0)
|
||||
last_error_code: Mapped[str | None] = mapped_column(String(64))
|
||||
locked_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||
locked_by: Mapped[str | None] = mapped_column(String(128))
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
|
||||
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
|
||||
|
||||
|
||||
class DeliveryOutbox(Base):
|
||||
__tablename__ = "delivery_outbox"
|
||||
__table_args__ = ({"schema": SCHEMA},)
|
||||
id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
|
||||
message_id: Mapped[uuid.UUID] = mapped_column(ForeignKey(f"{SCHEMA}.messages.id"), unique=True)
|
||||
external_chat_id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True))
|
||||
payload_json: Mapped[dict[str, Any]] = mapped_column(JSON)
|
||||
status: Mapped[str] = mapped_column(String(16), default="pending")
|
||||
attempt_count: Mapped[int] = mapped_column(Integer, default=0)
|
||||
next_attempt_at: Mapped[datetime] = mapped_column(DateTime(timezone=True))
|
||||
last_error_code: Mapped[str | None] = mapped_column(String(64))
|
||||
locked_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||
locked_by: Mapped[str | None] = mapped_column(String(128))
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
|
||||
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
|
||||
|
||||
|
||||
class IdempotencyRecord(Base):
|
||||
__tablename__ = "idempotency_records"
|
||||
__table_args__ = (
|
||||
UniqueConstraint("scope", "user_id", "idempotency_key"),
|
||||
{"schema": SCHEMA},
|
||||
)
|
||||
id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
|
||||
scope: Mapped[str] = mapped_column(String(128))
|
||||
user_id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True))
|
||||
idempotency_key: Mapped[str] = mapped_column(String(128))
|
||||
request_fingerprint: Mapped[str] = mapped_column(String(64))
|
||||
status: Mapped[str] = mapped_column(String(16))
|
||||
response_status: Mapped[int | None] = mapped_column(Integer)
|
||||
response_body_json: Mapped[dict[str, Any] | None] = mapped_column(JSON)
|
||||
resource_type: Mapped[str | None] = mapped_column(String(64))
|
||||
resource_id: Mapped[uuid.UUID | None] = mapped_column(UUID(as_uuid=True))
|
||||
expires_at: Mapped[datetime] = mapped_column(DateTime(timezone=True))
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
|
||||
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
|
||||
|
||||
|
||||
class OpenLinesInboxReceipt(Base):
|
||||
__tablename__ = "openlines_inbox_receipts"
|
||||
__table_args__ = (
|
||||
UniqueConstraint("event_id"),
|
||||
UniqueConstraint("external_chat_id", "bitrix_message_id"),
|
||||
{"schema": SCHEMA},
|
||||
)
|
||||
id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
|
||||
event_id: Mapped[str] = mapped_column(String(255))
|
||||
external_chat_id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True))
|
||||
bitrix_message_id: Mapped[str | None] = mapped_column(String(255))
|
||||
event_type: Mapped[str] = mapped_column(String(32))
|
||||
payload_fingerprint: Mapped[str] = mapped_column(String(64))
|
||||
status: Mapped[str] = mapped_column(String(16))
|
||||
message_id: Mapped[uuid.UUID | None] = mapped_column(UUID(as_uuid=True))
|
||||
last_error_code: Mapped[str | None] = mapped_column(String(64))
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
|
||||
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
|
||||
|
||||
|
||||
class AuditEvent(Base):
|
||||
__tablename__ = "audit_events"
|
||||
__table_args__ = ({"schema": SCHEMA},)
|
||||
id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
|
||||
event_type: Mapped[str] = mapped_column(String(128))
|
||||
actor_type: Mapped[str] = mapped_column(String(32))
|
||||
user_id: Mapped[uuid.UUID | None] = mapped_column(UUID(as_uuid=True))
|
||||
ux_session_id: Mapped[uuid.UUID | None] = mapped_column(UUID(as_uuid=True))
|
||||
request_id: Mapped[str] = mapped_column(String(64))
|
||||
trace_id: Mapped[str | None] = mapped_column(String(64))
|
||||
resource_type: Mapped[str | None] = mapped_column(String(64))
|
||||
resource_id: Mapped[uuid.UUID | None] = mapped_column(UUID(as_uuid=True))
|
||||
ip: Mapped[str | None] = mapped_column(INET)
|
||||
user_agent_hash: Mapped[str | None] = mapped_column(String(64))
|
||||
outcome: Mapped[str] = mapped_column(String(32))
|
||||
metadata_json: Mapped[dict[str, Any]] = mapped_column(JSON, default=dict)
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
|
||||
|
||||
|
||||
class AppSetting(Base):
|
||||
__tablename__ = "app_settings"
|
||||
__table_args__ = ({"schema": SCHEMA},)
|
||||
setting_key: Mapped[str] = mapped_column(String(255), primary_key=True)
|
||||
setting_value: Mapped[str] = mapped_column(Text)
|
||||
value_type: Mapped[str] = mapped_column(String(32))
|
||||
is_public: Mapped[bool] = mapped_column(Boolean, default=False)
|
||||
description: Mapped[str | None] = mapped_column(Text)
|
||||
record_status: Mapped[str] = mapped_column(String(1), default="A")
|
||||
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
|
||||
|
||||
|
||||
class TextResource(Common, Base):
|
||||
__tablename__ = "text_resources"
|
||||
__table_args__ = (
|
||||
UniqueConstraint("mnemonic", "locale"),
|
||||
{"schema": SCHEMA},
|
||||
)
|
||||
mnemonic: Mapped[str] = mapped_column(String(255))
|
||||
locale: Mapped[str] = mapped_column(String(16), default="ru")
|
||||
text_value: Mapped[str] = mapped_column(Text)
|
||||
sort_order: Mapped[int] = mapped_column(Integer, default=0)
|
||||
|
||||
|
||||
class PopularQuestion(Common, Base):
|
||||
__tablename__ = "popular_questions"
|
||||
__table_args__ = (
|
||||
UniqueConstraint("mnemonic", "locale"),
|
||||
{"schema": SCHEMA},
|
||||
)
|
||||
mnemonic: Mapped[str] = mapped_column(String(255))
|
||||
locale: Mapped[str] = mapped_column(String(16), default="ru")
|
||||
question_text: Mapped[str] = mapped_column(Text)
|
||||
sort_order: Mapped[int] = mapped_column(Integer, default=0)
|
||||
|
||||
|
||||
class SyncQueue(Base):
|
||||
__tablename__ = "sync_queue"
|
||||
__table_args__ = ({"schema": SCHEMA},)
|
||||
id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
|
||||
task_type: Mapped[str] = mapped_column(String(64))
|
||||
entity_type: Mapped[str] = mapped_column(String(64))
|
||||
entity_id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True))
|
||||
dedup_key: Mapped[str] = mapped_column(String(255), unique=True)
|
||||
payload_json: Mapped[dict[str, Any]] = mapped_column(JSON)
|
||||
status: Mapped[str] = mapped_column(String(16), default="pending")
|
||||
attempt_count: Mapped[int] = mapped_column(Integer, default=0)
|
||||
next_attempt_at: Mapped[datetime] = mapped_column(
|
||||
DateTime(timezone=True), server_default=func.now()
|
||||
)
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
|
||||
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
|
||||
|
||||
|
||||
class EntityExternalMapping(Base):
|
||||
__tablename__ = "entity_external_mapping"
|
||||
__table_args__ = (UniqueConstraint("entity_type", "entity_id"), {"schema": SCHEMA})
|
||||
id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
|
||||
entity_type: Mapped[str] = mapped_column(String(64))
|
||||
entity_id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True))
|
||||
external_id: Mapped[str] = mapped_column(String(128))
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
|
||||
|
||||
|
||||
class Database:
|
||||
def __init__(self, url: str) -> None:
|
||||
self.engine: AsyncEngine = create_async_engine(url, pool_pre_ping=True)
|
||||
self.sessions = async_sessionmaker(self.engine, expire_on_commit=False)
|
||||
|
||||
async def session(self) -> AsyncIterator[AsyncSession]:
|
||||
async with self.sessions() as session:
|
||||
yield session
|
||||
|
||||
async def close(self) -> None:
|
||||
await self.engine.dispose()
|
||||
@@ -0,0 +1,338 @@
|
||||
import asyncio
|
||||
import hashlib
|
||||
import ipaddress
|
||||
import json
|
||||
import socket
|
||||
import time
|
||||
import uuid
|
||||
from dataclasses import dataclass
|
||||
from typing import Any
|
||||
from urllib.parse import urlparse
|
||||
|
||||
import boto3
|
||||
import httpx
|
||||
from botocore.config import Config
|
||||
from redis.asyncio import Redis
|
||||
|
||||
from app.settings import Settings
|
||||
|
||||
RATE_LIMIT_LUA = """
|
||||
local current = redis.call('INCR', KEYS[1])
|
||||
if current == 1 then redis.call('EXPIRE', KEYS[1], ARGV[1]) end
|
||||
local ttl = redis.call('TTL', KEYS[1])
|
||||
return {current, ttl}
|
||||
"""
|
||||
|
||||
|
||||
class DependencyFailure(Exception):
|
||||
def __init__(self, code: str = "dependency_unavailable", timeout: bool = False) -> None:
|
||||
self.code = code
|
||||
self.timeout = timeout
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class CircuitBreaker:
|
||||
threshold: int
|
||||
open_seconds: float
|
||||
failures: int = 0
|
||||
opened_at: float | None = None
|
||||
|
||||
def allow(self) -> bool:
|
||||
if self.opened_at is None:
|
||||
return True
|
||||
if time.monotonic() - self.opened_at >= self.open_seconds:
|
||||
self.opened_at = None
|
||||
self.failures = max(0, self.threshold - 1)
|
||||
return True
|
||||
return False
|
||||
|
||||
def success(self) -> None:
|
||||
self.failures = 0
|
||||
self.opened_at = None
|
||||
|
||||
def failure(self) -> None:
|
||||
self.failures += 1
|
||||
if self.failures >= self.threshold:
|
||||
self.opened_at = time.monotonic()
|
||||
|
||||
|
||||
class RateLimiter:
|
||||
def __init__(self, redis: Redis) -> None:
|
||||
self.redis = redis
|
||||
|
||||
async def consume(self, key: str, limit: int, window: int) -> int:
|
||||
try:
|
||||
count, ttl = await self.redis.eval(RATE_LIMIT_LUA, 1, key, window)
|
||||
except Exception as exc:
|
||||
raise DependencyFailure() from exc
|
||||
if int(count) > limit:
|
||||
return max(1, int(ttl))
|
||||
return 0
|
||||
|
||||
@staticmethod
|
||||
def key(identity_type: str, identity: str, route: str, window: int) -> str:
|
||||
safe_identity = hashlib.sha256(identity.encode()).hexdigest()[:32]
|
||||
bucket = int(time.time()) // window
|
||||
return f"han:api:rl:{identity_type}:{safe_identity}:{route}:{bucket}"
|
||||
|
||||
|
||||
class RedisIdempotency:
|
||||
def __init__(self, redis: Redis) -> None:
|
||||
self.redis = redis
|
||||
|
||||
async def get(self, scope: str, user_id: uuid.UUID, key: str) -> dict[str, Any] | None:
|
||||
key_hash = hashlib.sha256(key.encode()).hexdigest()
|
||||
raw = await self.redis.get(f"han:api:idem:{scope}:{user_id}:{key_hash}")
|
||||
return json.loads(raw) if raw else None
|
||||
|
||||
async def put(self, scope: str, user_id: uuid.UUID, key: str, value: dict[str, Any]) -> None:
|
||||
key_hash = hashlib.sha256(key.encode()).hexdigest()
|
||||
await self.redis.set(
|
||||
f"han:api:idem:{scope}:{user_id}:{key_hash}",
|
||||
json.dumps(value, separators=(",", ":"), default=str),
|
||||
ex=86400,
|
||||
)
|
||||
|
||||
|
||||
class SafetyClient:
|
||||
def __init__(self, settings: Settings, http: httpx.AsyncClient) -> None:
|
||||
self.settings = settings
|
||||
self.http = http
|
||||
self.breaker = CircuitBreaker(
|
||||
settings.message_safety_circuit_failure_threshold,
|
||||
settings.message_safety_circuit_open_sec,
|
||||
)
|
||||
|
||||
async def check(self, payload: dict[str, Any], request_id: str) -> dict[str, Any]:
|
||||
return await self._call(
|
||||
"POST",
|
||||
"/internal/safety/v1/messages/check",
|
||||
request_id,
|
||||
json=payload,
|
||||
timeout=self.settings.message_safety_post_timeout_sec,
|
||||
)
|
||||
|
||||
async def poll(self, task_id: str, request_id: str) -> dict[str, Any]:
|
||||
return await self._call(
|
||||
"GET",
|
||||
f"/internal/safety/v1/messages/tasks/{task_id}",
|
||||
request_id,
|
||||
timeout=2,
|
||||
)
|
||||
|
||||
async def _call(self, method: str, path: str, request_id: str, **kwargs: Any) -> dict[str, Any]:
|
||||
if not self.breaker.allow():
|
||||
raise DependencyFailure()
|
||||
headers = {
|
||||
"X-Service-Token": self.settings.message_safety_service_token.get_secret_value(),
|
||||
"X-Request-ID": request_id,
|
||||
}
|
||||
try:
|
||||
response = await self.http.request(
|
||||
method,
|
||||
f"{str(self.settings.message_safety_url).rstrip('/')}{path}",
|
||||
headers=headers,
|
||||
**kwargs,
|
||||
)
|
||||
except httpx.TimeoutException as exc:
|
||||
self.breaker.failure()
|
||||
raise DependencyFailure(timeout=True) from exc
|
||||
except httpx.HTTPError as exc:
|
||||
self.breaker.failure()
|
||||
raise DependencyFailure() from exc
|
||||
if response.status_code == 401 or response.status_code >= 500:
|
||||
self.breaker.failure()
|
||||
raise DependencyFailure()
|
||||
try:
|
||||
body = response.json()
|
||||
except ValueError as exc:
|
||||
self.breaker.failure()
|
||||
raise DependencyFailure() from exc
|
||||
if not isinstance(body, dict):
|
||||
self.breaker.failure()
|
||||
raise DependencyFailure()
|
||||
self.breaker.success()
|
||||
body["_status"] = response.status_code
|
||||
return body
|
||||
|
||||
|
||||
class OpenLinesClient:
|
||||
def __init__(self, settings: Settings, http: httpx.AsyncClient) -> None:
|
||||
self.settings = settings
|
||||
self.http = http
|
||||
self.breaker = CircuitBreaker(
|
||||
settings.bitrix_local_app_circuit_failure_threshold,
|
||||
settings.bitrix_local_app_circuit_open_sec,
|
||||
)
|
||||
|
||||
async def send(
|
||||
self, message_id: uuid.UUID, payload: dict[str, Any], request_id: str
|
||||
) -> dict[str, Any]:
|
||||
if not self.breaker.allow():
|
||||
raise DependencyFailure()
|
||||
try:
|
||||
response = await self.http.post(
|
||||
f"{str(self.settings.bitrix_local_app_base_url).rstrip('/')}"
|
||||
"/internal/openlines/v1/messages",
|
||||
json=payload,
|
||||
headers={
|
||||
"Authorization": "Bearer "
|
||||
+ self.settings.bitrix_local_app_internal_token.get_secret_value(),
|
||||
"Idempotency-Key": str(message_id),
|
||||
"X-Request-ID": request_id,
|
||||
},
|
||||
timeout=self.settings.bitrix_local_app_http_timeout_sec,
|
||||
)
|
||||
response.raise_for_status()
|
||||
except httpx.TimeoutException as exc:
|
||||
self.breaker.failure()
|
||||
raise DependencyFailure(timeout=True) from exc
|
||||
except httpx.HTTPError as exc:
|
||||
self.breaker.failure()
|
||||
raise DependencyFailure() from exc
|
||||
self.breaker.success()
|
||||
return response.json()
|
||||
|
||||
async def ready(self) -> bool:
|
||||
try:
|
||||
response = await self.http.get(
|
||||
f"{str(self.settings.bitrix_local_app_base_url).rstrip('/')}"
|
||||
"/internal/openlines/v1/status",
|
||||
headers={
|
||||
"Authorization": "Bearer "
|
||||
+ self.settings.bitrix_local_app_internal_token.get_secret_value()
|
||||
},
|
||||
timeout=2,
|
||||
)
|
||||
return response.is_success
|
||||
except httpx.HTTPError:
|
||||
return False
|
||||
|
||||
|
||||
async def fresh_openlines_payload(
|
||||
payload: dict[str, Any], s3: "S3Client"
|
||||
) -> dict[str, Any]:
|
||||
result = json.loads(json.dumps(payload, default=str))
|
||||
for file in result.get("message", {}).get("files", []):
|
||||
bucket = file.pop("_storage_bucket")
|
||||
key = file.pop("_object_key")
|
||||
file["download_url"] = await s3.presign_get(bucket, key)
|
||||
return result
|
||||
|
||||
|
||||
class S3Client:
|
||||
def __init__(self, settings: Settings) -> None:
|
||||
self.settings = settings
|
||||
self.client = boto3.client(
|
||||
"s3",
|
||||
endpoint_url=str(settings.selectel_s3_endpoint_url),
|
||||
aws_access_key_id=settings.selectel_s3_access_key.get_secret_value(),
|
||||
aws_secret_access_key=settings.selectel_s3_secret_key.get_secret_value(),
|
||||
config=Config(connect_timeout=3, read_timeout=10, retries={"max_attempts": 2}),
|
||||
)
|
||||
|
||||
async def ready(self) -> bool:
|
||||
try:
|
||||
for bucket in (
|
||||
self.settings.selectel_s3_bucket_quarantine,
|
||||
self.settings.selectel_s3_bucket_attachments,
|
||||
self.settings.selectel_s3_bucket_documents,
|
||||
):
|
||||
await asyncio.to_thread(self.client.head_bucket, Bucket=bucket)
|
||||
return True
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
async def presign_put(self, key: str, mime: str, ttl: int) -> str:
|
||||
return await asyncio.to_thread(
|
||||
self.client.generate_presigned_url,
|
||||
"put_object",
|
||||
Params={
|
||||
"Bucket": self.settings.selectel_s3_bucket_quarantine,
|
||||
"Key": key,
|
||||
"ContentType": mime,
|
||||
},
|
||||
ExpiresIn=ttl,
|
||||
)
|
||||
|
||||
async def presign_get(self, bucket: str, key: str, ttl: int = 300) -> str:
|
||||
return await asyncio.to_thread(
|
||||
self.client.generate_presigned_url,
|
||||
"get_object",
|
||||
Params={"Bucket": bucket, "Key": key},
|
||||
ExpiresIn=ttl,
|
||||
)
|
||||
|
||||
async def head(self, bucket: str, key: str) -> dict[str, Any]:
|
||||
return await asyncio.to_thread(self.client.head_object, Bucket=bucket, Key=key)
|
||||
|
||||
async def promote(self, source_key: str, destination_key: str) -> None:
|
||||
await asyncio.to_thread(
|
||||
self.client.copy_object,
|
||||
Bucket=self.settings.selectel_s3_bucket_attachments,
|
||||
Key=destination_key,
|
||||
CopySource={
|
||||
"Bucket": self.settings.selectel_s3_bucket_quarantine,
|
||||
"Key": source_key,
|
||||
},
|
||||
)
|
||||
await asyncio.to_thread(
|
||||
self.client.delete_object,
|
||||
Bucket=self.settings.selectel_s3_bucket_quarantine,
|
||||
Key=source_key,
|
||||
)
|
||||
|
||||
async def delete_quarantine(self, key: str) -> None:
|
||||
await asyncio.to_thread(
|
||||
self.client.delete_object,
|
||||
Bucket=self.settings.selectel_s3_bucket_quarantine,
|
||||
Key=key,
|
||||
)
|
||||
|
||||
async def upload_inbound(
|
||||
self,
|
||||
http: httpx.AsyncClient,
|
||||
download_url: str,
|
||||
destination_key: str,
|
||||
mime_type: str,
|
||||
max_bytes: int,
|
||||
) -> tuple[int, str]:
|
||||
parsed = urlparse(download_url)
|
||||
if parsed.scheme != "https" or not parsed.hostname:
|
||||
raise DependencyFailure("unsafe_inbound_url")
|
||||
try:
|
||||
addresses = await asyncio.to_thread(
|
||||
socket.getaddrinfo, parsed.hostname, parsed.port or 443, type=socket.SOCK_STREAM
|
||||
)
|
||||
except OSError as exc:
|
||||
raise DependencyFailure("unsafe_inbound_url") from exc
|
||||
if any(
|
||||
ipaddress.ip_address(address[4][0]).is_private
|
||||
or ipaddress.ip_address(address[4][0]).is_loopback
|
||||
or ipaddress.ip_address(address[4][0]).is_link_local
|
||||
or ipaddress.ip_address(address[4][0]).is_reserved
|
||||
for address in addresses
|
||||
):
|
||||
raise DependencyFailure("unsafe_inbound_url")
|
||||
data = bytearray()
|
||||
try:
|
||||
async with http.stream(
|
||||
"GET", download_url, timeout=10, follow_redirects=False
|
||||
) as response:
|
||||
response.raise_for_status()
|
||||
if response.headers.get("content-type", "").split(";")[0] != mime_type:
|
||||
raise DependencyFailure("inbound_mime_mismatch")
|
||||
async for chunk in response.aiter_bytes():
|
||||
data.extend(chunk)
|
||||
if len(data) > max_bytes:
|
||||
raise DependencyFailure("inbound_file_too_large")
|
||||
except httpx.HTTPError as exc:
|
||||
raise DependencyFailure() from exc
|
||||
await asyncio.to_thread(
|
||||
self.client.put_object,
|
||||
Bucket=self.settings.selectel_s3_bucket_attachments,
|
||||
Key=destination_key,
|
||||
Body=bytes(data),
|
||||
ContentType=mime_type,
|
||||
)
|
||||
return len(data), hashlib.sha256(data).hexdigest()
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,65 @@
|
||||
import asyncio
|
||||
import json
|
||||
import uuid
|
||||
from collections.abc import AsyncIterator
|
||||
from contextlib import suppress
|
||||
from typing import Any
|
||||
|
||||
from redis.asyncio import Redis
|
||||
|
||||
CHANNEL_PREFIX = "han:realtime:dialog:"
|
||||
|
||||
|
||||
class LocalFanout:
|
||||
def __init__(self) -> None:
|
||||
self._queues: set[asyncio.Queue[dict[str, Any]]] = set()
|
||||
|
||||
async def publish(self, event: dict[str, Any]) -> None:
|
||||
for queue in tuple(self._queues):
|
||||
with suppress(asyncio.QueueFull):
|
||||
queue.put_nowait(event)
|
||||
|
||||
async def subscribe(self) -> AsyncIterator[dict[str, Any]]:
|
||||
queue: asyncio.Queue[dict[str, Any]] = asyncio.Queue(maxsize=256)
|
||||
self._queues.add(queue)
|
||||
try:
|
||||
while True:
|
||||
yield await queue.get()
|
||||
finally:
|
||||
self._queues.discard(queue)
|
||||
|
||||
|
||||
class RealtimeFanout:
|
||||
def __init__(self, redis: Redis, local: LocalFanout | None = None) -> None:
|
||||
self.redis = redis
|
||||
self.local = local or LocalFanout()
|
||||
|
||||
async def publish(self, event: dict[str, Any]) -> None:
|
||||
event = {"event_id": str(uuid.uuid4()), **event}
|
||||
channel = CHANNEL_PREFIX + str(event["dialog_id"])
|
||||
try:
|
||||
await self.redis.publish(channel, json.dumps(event, default=str, separators=(",", ":")))
|
||||
except Exception:
|
||||
await self.local.publish(event)
|
||||
|
||||
async def events(self, dialog_ids: set[uuid.UUID]) -> AsyncIterator[dict[str, Any]]:
|
||||
channels = [CHANNEL_PREFIX + str(dialog_id) for dialog_id in dialog_ids]
|
||||
pubsub = self.redis.pubsub()
|
||||
try:
|
||||
await pubsub.subscribe(*channels)
|
||||
except Exception:
|
||||
await pubsub.aclose()
|
||||
async for event in self.local.subscribe():
|
||||
if uuid.UUID(str(event["dialog_id"])) in dialog_ids:
|
||||
yield event
|
||||
return
|
||||
try:
|
||||
while True:
|
||||
message = await pubsub.get_message(ignore_subscribe_messages=True, timeout=1)
|
||||
if message:
|
||||
yield json.loads(message["data"])
|
||||
else:
|
||||
await asyncio.sleep(0)
|
||||
finally:
|
||||
await pubsub.unsubscribe(*channels)
|
||||
await pubsub.aclose()
|
||||
@@ -0,0 +1,148 @@
|
||||
import hashlib
|
||||
import hmac
|
||||
import json
|
||||
import uuid
|
||||
from base64 import urlsafe_b64decode, urlsafe_b64encode
|
||||
from datetime import datetime
|
||||
from enum import StrEnum
|
||||
from typing import Annotated, Literal
|
||||
|
||||
from pydantic import BaseModel, ConfigDict, Field, HttpUrl, model_validator
|
||||
|
||||
|
||||
class StrictModel(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class Device(StrictModel):
|
||||
platform: Literal["ios", "android", "web"]
|
||||
app_version: str = Field(min_length=1, max_length=64)
|
||||
device_id: str | None = Field(default=None, max_length=255)
|
||||
|
||||
|
||||
class ConsentChoice(StrictModel):
|
||||
accepted: bool
|
||||
version: str = Field(min_length=1, max_length=64)
|
||||
|
||||
|
||||
class ConsentSet(StrictModel):
|
||||
personal_data: ConsentChoice
|
||||
user_agreement: ConsentChoice
|
||||
marketing: ConsentChoice
|
||||
|
||||
|
||||
class BootstrapRequest(StrictModel):
|
||||
consents: ConsentSet
|
||||
device: Device
|
||||
|
||||
|
||||
class ConsentsRequest(StrictModel):
|
||||
consents: ConsentSet
|
||||
|
||||
|
||||
class SessionStartRequest(StrictModel):
|
||||
start_reason: Literal["first_launch", "cold_start", "idle_timeout"]
|
||||
device: Device
|
||||
|
||||
|
||||
class TextMessageRequest(StrictModel):
|
||||
content_kind: Literal["text"]
|
||||
text: str = Field(min_length=1, max_length=4000)
|
||||
|
||||
|
||||
class FileMessageRequest(StrictModel):
|
||||
content_kind: Literal["file"]
|
||||
attachment_id: uuid.UUID
|
||||
checksum: str = Field(pattern=r"^sha256:[0-9a-f]{64}$")
|
||||
|
||||
|
||||
MessageRequest = Annotated[
|
||||
TextMessageRequest | FileMessageRequest, Field(discriminator="content_kind")
|
||||
]
|
||||
|
||||
|
||||
class AttachmentInitRequest(StrictModel):
|
||||
file_name: str = Field(min_length=1, max_length=255)
|
||||
mime_type: str = Field(min_length=1, max_length=128)
|
||||
size_bytes: int = Field(gt=0)
|
||||
|
||||
|
||||
class AttachmentCompleteRequest(StrictModel):
|
||||
checksum: str = Field(pattern=r"^sha256:[0-9a-f]{64}$")
|
||||
|
||||
|
||||
class OpenLinesFile(StrictModel):
|
||||
name: str = Field(min_length=1, max_length=255)
|
||||
mime_type: str = Field(min_length=1, max_length=128)
|
||||
size_bytes: int = Field(gt=0)
|
||||
download_url: HttpUrl
|
||||
|
||||
|
||||
class OpenLinesMessage(StrictModel):
|
||||
text: str = Field(default="", max_length=4000)
|
||||
files: list[OpenLinesFile] = Field(default_factory=list, max_length=1)
|
||||
|
||||
@model_validator(mode="after")
|
||||
def non_empty(self) -> "OpenLinesMessage":
|
||||
if not self.text.strip() and not self.files:
|
||||
raise ValueError("message must contain text or file")
|
||||
return self
|
||||
|
||||
|
||||
class OpenLinesInbox(StrictModel):
|
||||
event_id: str = Field(min_length=1, max_length=255)
|
||||
event_type: Literal["message.new", "dialog.closed"]
|
||||
external_chat_id: uuid.UUID
|
||||
bitrix_message_id: str | None = Field(default=None, max_length=255)
|
||||
occurred_at: datetime
|
||||
message: OpenLinesMessage | None = None
|
||||
|
||||
@model_validator(mode="after")
|
||||
def event_shape(self) -> "OpenLinesInbox":
|
||||
if self.event_type == "message.new" and (
|
||||
not self.bitrix_message_id or self.message is None
|
||||
):
|
||||
raise ValueError("message.new requires bitrix_message_id and message")
|
||||
return self
|
||||
|
||||
|
||||
class DialogStatus(StrEnum):
|
||||
OPEN = "open"
|
||||
WAITING_COMPANY = "waiting_for_company"
|
||||
WAITING_CLIENT = "waiting_for_client"
|
||||
CLOSED = "closed"
|
||||
|
||||
|
||||
def canonical_fingerprint(
|
||||
method: str, route: str, path_params: dict[str, str], body: object, user_id: uuid.UUID
|
||||
) -> str:
|
||||
value = {
|
||||
"method": method.upper(),
|
||||
"route": route,
|
||||
"path": dict(sorted(path_params.items())),
|
||||
"body": body,
|
||||
"user_id": str(user_id),
|
||||
}
|
||||
return hashlib.sha256(
|
||||
json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=False).encode()
|
||||
).hexdigest()
|
||||
|
||||
|
||||
def encode_cursor(data: dict[str, str], secret: bytes) -> str:
|
||||
payload = json.dumps({"v": 1, **data}, sort_keys=True, separators=(",", ":")).encode()
|
||||
signature = hmac.digest(secret, payload, "sha256")
|
||||
return urlsafe_b64encode(payload + signature).decode().rstrip("=")
|
||||
|
||||
|
||||
def decode_cursor(value: str, secret: bytes) -> dict[str, str]:
|
||||
try:
|
||||
raw = urlsafe_b64decode(value + "=" * (-len(value) % 4))
|
||||
payload, signature = raw[:-32], raw[-32:]
|
||||
if not hmac.compare_digest(signature, hmac.digest(secret, payload, "sha256")):
|
||||
raise ValueError("invalid cursor")
|
||||
decoded = json.loads(payload)
|
||||
if decoded.pop("v") != 1:
|
||||
raise ValueError("unsupported cursor")
|
||||
return decoded
|
||||
except (ValueError, KeyError, json.JSONDecodeError) as exc:
|
||||
raise ValueError("invalid cursor") from exc
|
||||
@@ -0,0 +1,930 @@
|
||||
import hashlib
|
||||
import json
|
||||
import re
|
||||
import unicodedata
|
||||
import uuid
|
||||
from dataclasses import dataclass
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from pathlib import PurePath
|
||||
from typing import Any
|
||||
|
||||
import httpx
|
||||
from sqlalchemy import func, select
|
||||
from sqlalchemy.dialects.postgresql import insert
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.auth import Principal
|
||||
from app.db import (
|
||||
AppSetting,
|
||||
AuditEvent,
|
||||
ClientProfile,
|
||||
DeliveryOutbox,
|
||||
Dialog,
|
||||
Document,
|
||||
IdempotencyRecord,
|
||||
Message,
|
||||
MessageAttachment,
|
||||
OpenLinesInboxReceipt,
|
||||
SafetyTask,
|
||||
UserConsent,
|
||||
UserIdentity,
|
||||
UxSession,
|
||||
)
|
||||
from app.integrations import (
|
||||
DependencyFailure,
|
||||
OpenLinesClient,
|
||||
S3Client,
|
||||
SafetyClient,
|
||||
fresh_openlines_payload,
|
||||
)
|
||||
from app.realtime import RealtimeFanout
|
||||
from app.schemas import (
|
||||
AttachmentCompleteRequest,
|
||||
AttachmentInitRequest,
|
||||
BootstrapRequest,
|
||||
ConsentsRequest,
|
||||
FileMessageRequest,
|
||||
MessageRequest,
|
||||
OpenLinesInbox,
|
||||
SessionStartRequest,
|
||||
encode_cursor,
|
||||
)
|
||||
from app.settings import Settings
|
||||
|
||||
|
||||
class DomainError(Exception):
|
||||
def __init__(self, code: str, status: int, message: str, details: dict[str, Any] | None = None):
|
||||
self.code, self.status, self.message = code, status, message
|
||||
self.details = details or {}
|
||||
|
||||
|
||||
REQUIRED_SETTINGS = {
|
||||
"auth.phone.enabled",
|
||||
"auth.password.enabled",
|
||||
"otp.phone.max_send_attempts_per_24h",
|
||||
"otp.phone.min_seconds_between_attempts",
|
||||
"operator.call.phone",
|
||||
"consent.personal_data.required",
|
||||
"consent.personal_data.document_url",
|
||||
"consent.personal_data.version",
|
||||
"consent.user_agreement.required",
|
||||
"consent.user_agreement.document_url",
|
||||
"consent.user_agreement.version",
|
||||
"consent.marketing.required",
|
||||
"consent.marketing.version",
|
||||
"chat.attachments.allowed_extensions",
|
||||
"chat.attachments.allowed_mime_types",
|
||||
"chat.attachments.max_size_mb",
|
||||
"chat.attachments.presigned_upload_ttl_seconds",
|
||||
"rate_limit.message_send.per_user",
|
||||
"rate_limit.message_send.per_dialog",
|
||||
"rate_limit.download_url.per_user",
|
||||
"rate_limit.public_endpoints.per_ip",
|
||||
"ux.session.idle_timeout_minutes",
|
||||
"security.cors.allowed_origins",
|
||||
"security.public_cache.max_age_seconds",
|
||||
}
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class SettingsSnapshot:
|
||||
values: dict[str, str]
|
||||
version: str
|
||||
|
||||
def boolean(self, key: str) -> bool:
|
||||
return self.values[key].lower() == "true"
|
||||
|
||||
def integer(self, key: str) -> int:
|
||||
return int(self.values[key])
|
||||
|
||||
def strings(self, key: str) -> list[str]:
|
||||
return [item.strip() for item in self.values[key].split(",") if item.strip()]
|
||||
|
||||
def limit(self, key: str) -> tuple[int, int]:
|
||||
amount, period = self.values[key].split("/", 1)
|
||||
windows = {"second": 1, "minute": 60, "hour": 3600, "day": 86400}
|
||||
return int(amount), windows[period]
|
||||
|
||||
|
||||
async def load_settings(session: AsyncSession) -> SettingsSnapshot:
|
||||
rows = (
|
||||
await session.execute(select(AppSetting).where(AppSetting.record_status == "A"))
|
||||
).scalars()
|
||||
values = {row.setting_key: row.setting_value for row in rows}
|
||||
missing = REQUIRED_SETTINGS - values.keys()
|
||||
if missing:
|
||||
raise DomainError(
|
||||
"dependency_unavailable",
|
||||
503,
|
||||
"Required settings are unavailable",
|
||||
{"missing": sorted(missing)},
|
||||
)
|
||||
version = hashlib.sha256(json.dumps(values, sort_keys=True).encode()).hexdigest()[:24]
|
||||
return SettingsSnapshot(values, version)
|
||||
|
||||
|
||||
async def resolve_user(session: AsyncSession, principal: Principal) -> UserIdentity:
|
||||
user = (
|
||||
await session.execute(
|
||||
select(UserIdentity).where(
|
||||
UserIdentity.keycloak_sub == principal.subject, UserIdentity.record_status == "A"
|
||||
)
|
||||
)
|
||||
).scalar_one_or_none()
|
||||
if user is None:
|
||||
raise DomainError("resource_state_conflict", 409, "Bootstrap required")
|
||||
return user
|
||||
|
||||
|
||||
def audit(
|
||||
event_type: str,
|
||||
request_id: str,
|
||||
user_id: uuid.UUID | None,
|
||||
resource_type: str | None = None,
|
||||
resource_id: uuid.UUID | None = None,
|
||||
ux_session_id: uuid.UUID | None = None,
|
||||
metadata: dict[str, Any] | None = None,
|
||||
) -> AuditEvent:
|
||||
return AuditEvent(
|
||||
event_type=event_type,
|
||||
actor_type="user" if user_id else "service",
|
||||
user_id=user_id,
|
||||
ux_session_id=ux_session_id,
|
||||
request_id=request_id,
|
||||
resource_type=resource_type,
|
||||
resource_id=resource_id,
|
||||
outcome="success",
|
||||
metadata_json=metadata or {},
|
||||
)
|
||||
|
||||
|
||||
def validate_consents(consents: Any, snapshot: SettingsSnapshot) -> None:
|
||||
for name in ("personal_data", "user_agreement", "marketing"):
|
||||
choice = getattr(consents, name)
|
||||
if choice.version != snapshot.values[f"consent.{name}.version"]:
|
||||
raise DomainError(
|
||||
"validation_error", 400, "Consent version is not current", {"field": name}
|
||||
)
|
||||
if snapshot.boolean(f"consent.{name}.required") and not choice.accepted:
|
||||
raise DomainError("consents_required", 403, "Required consents must be accepted")
|
||||
|
||||
|
||||
async def bootstrap(
|
||||
session: AsyncSession,
|
||||
principal: Principal,
|
||||
body: BootstrapRequest,
|
||||
snapshot: SettingsSnapshot,
|
||||
request_id: str,
|
||||
) -> dict[str, Any]:
|
||||
if not principal.phone_number:
|
||||
raise DomainError("phone_claim_missing", 400, "Verified phone claim is missing")
|
||||
validate_consents(body.consents, snapshot)
|
||||
now = datetime.now(UTC)
|
||||
statement = (
|
||||
insert(UserIdentity)
|
||||
.values(
|
||||
id=uuid.uuid4(),
|
||||
keycloak_sub=principal.subject,
|
||||
phone_number=principal.phone_number,
|
||||
last_login_at=now,
|
||||
)
|
||||
.on_conflict_do_update(
|
||||
index_elements=[UserIdentity.keycloak_sub],
|
||||
set_={"phone_number": principal.phone_number, "last_login_at": now, "updated_at": now},
|
||||
)
|
||||
.returning(UserIdentity.id)
|
||||
)
|
||||
user_id = (await session.execute(statement)).scalar_one()
|
||||
await session.execute(
|
||||
insert(ClientProfile)
|
||||
.values(id=uuid.uuid4(), user_id=user_id)
|
||||
.on_conflict_do_nothing(index_elements=[ClientProfile.user_id])
|
||||
)
|
||||
for consent_type in ("personal_data", "user_agreement", "marketing"):
|
||||
choice = getattr(body.consents, consent_type)
|
||||
await session.execute(
|
||||
insert(UserConsent)
|
||||
.values(
|
||||
id=uuid.uuid4(),
|
||||
user_id=user_id,
|
||||
consent_type=consent_type,
|
||||
document_version=choice.version,
|
||||
accepted=choice.accepted,
|
||||
accepted_at=now,
|
||||
)
|
||||
.on_conflict_do_nothing(
|
||||
index_elements=[
|
||||
UserConsent.user_id,
|
||||
UserConsent.consent_type,
|
||||
UserConsent.document_version,
|
||||
]
|
||||
)
|
||||
)
|
||||
session.add(audit("auth.bootstrap", request_id, user_id))
|
||||
await session.commit()
|
||||
return {"user_id": user_id, "profile_ready": True}
|
||||
|
||||
|
||||
async def record_consents(
|
||||
session: AsyncSession,
|
||||
user: UserIdentity,
|
||||
body: ConsentsRequest,
|
||||
snapshot: SettingsSnapshot,
|
||||
request_id: str,
|
||||
ux_session_id: uuid.UUID | None,
|
||||
) -> dict[str, Any]:
|
||||
validate_consents(body.consents, snapshot)
|
||||
now = datetime.now(UTC)
|
||||
versions: dict[str, str] = {}
|
||||
for consent_type in ("personal_data", "user_agreement", "marketing"):
|
||||
choice = getattr(body.consents, consent_type)
|
||||
versions[consent_type] = choice.version
|
||||
await session.execute(
|
||||
insert(UserConsent)
|
||||
.values(
|
||||
id=uuid.uuid4(),
|
||||
user_id=user.id,
|
||||
ux_session_id=ux_session_id,
|
||||
consent_type=consent_type,
|
||||
document_version=choice.version,
|
||||
accepted=choice.accepted,
|
||||
accepted_at=now,
|
||||
)
|
||||
.on_conflict_do_nothing()
|
||||
)
|
||||
session.add(audit("consent.recorded", request_id, user.id, ux_session_id=ux_session_id))
|
||||
await session.commit()
|
||||
return {"recorded_at": now, "versions": versions}
|
||||
|
||||
|
||||
async def start_session(
|
||||
session: AsyncSession, user: UserIdentity, body: SessionStartRequest, request_id: str
|
||||
) -> dict[str, Any]:
|
||||
now, session_id = datetime.now(UTC), uuid.uuid4()
|
||||
device_hash = (
|
||||
hashlib.sha256(body.device.device_id.encode()).hexdigest()
|
||||
if body.device.device_id
|
||||
else None
|
||||
)
|
||||
session.add(
|
||||
UxSession(
|
||||
id=session_id,
|
||||
user_id=user.id,
|
||||
start_reason=body.start_reason,
|
||||
platform=body.device.platform,
|
||||
app_version=body.device.app_version,
|
||||
device_id_hash=device_hash,
|
||||
started_at=now,
|
||||
)
|
||||
)
|
||||
session.add(audit("session_start", request_id, user.id, ux_session_id=session_id))
|
||||
await session.commit()
|
||||
return {"ux_session_id": session_id, "started_at": now}
|
||||
|
||||
|
||||
async def get_profile(session: AsyncSession, user: UserIdentity) -> dict[str, Any]:
|
||||
profile = (
|
||||
await session.execute(
|
||||
select(ClientProfile).where(
|
||||
ClientProfile.user_id == user.id, ClientProfile.record_status == "A"
|
||||
)
|
||||
)
|
||||
).scalar_one()
|
||||
document_count = (
|
||||
await session.scalar(
|
||||
select(func.count(Document.id)).where(
|
||||
Document.user_id == user.id, Document.record_status == "A"
|
||||
)
|
||||
)
|
||||
or 0
|
||||
)
|
||||
return {
|
||||
"user_id": user.id,
|
||||
"profile": {
|
||||
"personal_data": {
|
||||
"full_name": profile.full_name,
|
||||
"citizenship": profile.citizenship,
|
||||
"russian_phone": profile.russian_phone,
|
||||
"foreign_phone": profile.foreign_phone,
|
||||
"email": profile.email,
|
||||
},
|
||||
"documents": {"count": document_count},
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def dialog_dto(dialog: Dialog) -> dict[str, Any]:
|
||||
return {
|
||||
"dialog_id": dialog.id,
|
||||
"status": dialog.status,
|
||||
"last_message_preview": None,
|
||||
"unread_count": 0,
|
||||
"created_at": dialog.created_at,
|
||||
"updated_at": dialog.updated_at,
|
||||
}
|
||||
|
||||
|
||||
def message_dto(
|
||||
message: Message, attachments: list[MessageAttachment] | None = None
|
||||
) -> dict[str, Any]:
|
||||
return {
|
||||
"message_id": message.id,
|
||||
"dialog_id": message.dialog_id,
|
||||
"sender_type": message.sender_type,
|
||||
"content_kind": message.content_kind,
|
||||
"text": message.text,
|
||||
"attachments": [
|
||||
{
|
||||
"attachment_id": item.id,
|
||||
"file_name": item.safe_file_name,
|
||||
"mime_type": item.mime_type,
|
||||
"size_bytes": item.size_bytes,
|
||||
"scan_status": item.scan_status,
|
||||
}
|
||||
for item in (attachments or [])
|
||||
],
|
||||
"safety_status": message.safety_status,
|
||||
"delivery_status": message.delivery_status,
|
||||
"created_at": message.created_at,
|
||||
}
|
||||
|
||||
|
||||
def message_cursor(message: Message, settings: Settings) -> str:
|
||||
return encode_cursor(
|
||||
{"created_at": message.created_at.isoformat(), "id": str(message.id)},
|
||||
settings.cursor_hmac_secret.get_secret_value().encode(),
|
||||
)
|
||||
|
||||
|
||||
async def publish_message(
|
||||
fanout: RealtimeFanout,
|
||||
message: Message,
|
||||
settings: Settings,
|
||||
attachments: list[MessageAttachment] | None = None,
|
||||
) -> None:
|
||||
await fanout.publish(
|
||||
{
|
||||
"type": "message.new",
|
||||
"dialog_id": str(message.dialog_id),
|
||||
"message": message_dto(message, attachments),
|
||||
"cursor": message_cursor(message, settings),
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
async def publish_message_status(
|
||||
fanout: RealtimeFanout, message: Message, settings: Settings
|
||||
) -> None:
|
||||
await fanout.publish(
|
||||
{
|
||||
"type": "message.status",
|
||||
"dialog_id": str(message.dialog_id),
|
||||
"message_id": str(message.id),
|
||||
"safety_status": message.safety_status,
|
||||
"delivery_status": message.delivery_status,
|
||||
"cursor": message_cursor(message, settings),
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
async def publish_dialog_status(fanout: RealtimeFanout, dialog: Dialog) -> None:
|
||||
await fanout.publish(
|
||||
{"type": "dialog.status", "dialog_id": str(dialog.id), "status": dialog.status}
|
||||
)
|
||||
|
||||
|
||||
async def owned_dialog(session: AsyncSession, user_id: uuid.UUID, dialog_id: uuid.UUID) -> Dialog:
|
||||
dialog = (
|
||||
await session.execute(
|
||||
select(Dialog).where(
|
||||
Dialog.id == dialog_id, Dialog.user_id == user_id, Dialog.record_status == "A"
|
||||
)
|
||||
)
|
||||
).scalar_one_or_none()
|
||||
if dialog is None:
|
||||
raise DomainError("not_found", 404, "Resource was not found")
|
||||
return dialog
|
||||
|
||||
|
||||
async def create_dialog(
|
||||
session: AsyncSession, user: UserIdentity, request_id: str, idempotency_key: str
|
||||
) -> tuple[dict[str, Any], int]:
|
||||
scope = "dialogs.create"
|
||||
fingerprint = hashlib.sha256(f"POST:/api/v1/dialogs:{user.id}".encode()).hexdigest()
|
||||
record = (
|
||||
await session.execute(
|
||||
select(IdempotencyRecord).where(
|
||||
IdempotencyRecord.scope == scope,
|
||||
IdempotencyRecord.user_id == user.id,
|
||||
IdempotencyRecord.idempotency_key == idempotency_key,
|
||||
)
|
||||
)
|
||||
).scalar_one_or_none()
|
||||
if record:
|
||||
if record.request_fingerprint != fingerprint:
|
||||
raise DomainError("idempotency_key_reused", 409, "Idempotency key was reused")
|
||||
if record.status == "completed" and record.response_body_json:
|
||||
return record.response_body_json, record.response_status or 200
|
||||
existing = (
|
||||
await session.execute(
|
||||
select(Dialog).where(
|
||||
Dialog.user_id == user.id,
|
||||
Dialog.record_status == "A",
|
||||
Dialog.status.in_(["open", "waiting_for_company", "waiting_for_client"]),
|
||||
)
|
||||
)
|
||||
).scalar_one_or_none()
|
||||
if existing:
|
||||
result = dialog_dto(existing)
|
||||
session.add(
|
||||
IdempotencyRecord(
|
||||
scope=scope,
|
||||
user_id=user.id,
|
||||
idempotency_key=idempotency_key,
|
||||
request_fingerprint=fingerprint,
|
||||
status="completed",
|
||||
response_status=200,
|
||||
response_body_json=json.loads(json.dumps(result, default=str)),
|
||||
resource_type="dialog",
|
||||
resource_id=existing.id,
|
||||
expires_at=datetime.now(UTC) + timedelta(hours=24),
|
||||
)
|
||||
)
|
||||
await session.commit()
|
||||
return result, 200
|
||||
dialog = Dialog(id=uuid.uuid4(), user_id=user.id, status="open")
|
||||
session.add(dialog)
|
||||
session.add(audit("dialog.created", request_id, user.id, "dialog", dialog.id))
|
||||
result = dialog_dto(dialog)
|
||||
session.add(
|
||||
IdempotencyRecord(
|
||||
scope=scope,
|
||||
user_id=user.id,
|
||||
idempotency_key=idempotency_key,
|
||||
request_fingerprint=fingerprint,
|
||||
status="completed",
|
||||
response_status=201,
|
||||
response_body_json=json.loads(json.dumps(result, default=str)),
|
||||
resource_type="dialog",
|
||||
resource_id=dialog.id,
|
||||
expires_at=datetime.now(UTC) + timedelta(hours=24),
|
||||
)
|
||||
)
|
||||
await session.commit()
|
||||
await session.refresh(dialog)
|
||||
return dialog_dto(dialog), 201
|
||||
|
||||
|
||||
async def init_attachment(
|
||||
session: AsyncSession,
|
||||
user: UserIdentity,
|
||||
dialog_id: uuid.UUID,
|
||||
body: AttachmentInitRequest,
|
||||
snapshot: SettingsSnapshot,
|
||||
s3: S3Client,
|
||||
request_id: str,
|
||||
) -> dict[str, Any]:
|
||||
await owned_dialog(session, user.id, dialog_id)
|
||||
extension = PurePath(body.file_name).suffix.lower().lstrip(".")
|
||||
if (
|
||||
extension not in snapshot.strings("chat.attachments.allowed_extensions")
|
||||
or body.mime_type not in snapshot.strings("chat.attachments.allowed_mime_types")
|
||||
or body.size_bytes > snapshot.integer("chat.attachments.max_size_mb") * 1024 * 1024
|
||||
):
|
||||
raise DomainError("validation_error", 400, "File type or size is not allowed")
|
||||
attachment_id = uuid.uuid4()
|
||||
key = f"quarantine/users/{user.id}/dialogs/{dialog_id}/{attachment_id}"
|
||||
ttl = snapshot.integer("chat.attachments.presigned_upload_ttl_seconds")
|
||||
expires = datetime.now(UTC) + timedelta(seconds=ttl)
|
||||
safe_name = re.sub(r"[^A-Za-z0-9._-]", "_", unicodedata.normalize("NFKC", body.file_name))
|
||||
item = MessageAttachment(
|
||||
id=attachment_id,
|
||||
dialog_id=dialog_id,
|
||||
owner_user_id=user.id,
|
||||
direction="client_upload",
|
||||
original_file_name=body.file_name,
|
||||
safe_file_name=safe_name,
|
||||
mime_type=body.mime_type,
|
||||
size_bytes=body.size_bytes,
|
||||
scan_status="pending",
|
||||
storage_bucket=s3.settings.selectel_s3_bucket_quarantine,
|
||||
object_key=key,
|
||||
quarantine_object_key=key,
|
||||
upload_expires_at=expires,
|
||||
)
|
||||
session.add(item)
|
||||
session.add(
|
||||
audit("attachment.upload_initialized", request_id, user.id, "attachment", attachment_id)
|
||||
)
|
||||
await session.commit()
|
||||
url = await s3.presign_put(key, body.mime_type, ttl)
|
||||
return {
|
||||
"attachment_id": attachment_id,
|
||||
"upload_url": url,
|
||||
"upload_headers": {"Content-Type": body.mime_type},
|
||||
"expires_at": expires,
|
||||
}
|
||||
|
||||
|
||||
async def complete_attachment(
|
||||
session: AsyncSession,
|
||||
user: UserIdentity,
|
||||
dialog_id: uuid.UUID,
|
||||
attachment_id: uuid.UUID,
|
||||
body: AttachmentCompleteRequest,
|
||||
s3: S3Client,
|
||||
request_id: str,
|
||||
) -> dict[str, Any]:
|
||||
item = (
|
||||
await session.execute(
|
||||
select(MessageAttachment).where(
|
||||
MessageAttachment.id == attachment_id,
|
||||
MessageAttachment.dialog_id == dialog_id,
|
||||
MessageAttachment.owner_user_id == user.id,
|
||||
MessageAttachment.record_status == "A",
|
||||
)
|
||||
)
|
||||
).scalar_one_or_none()
|
||||
if item is None:
|
||||
raise DomainError("not_found", 404, "Resource was not found")
|
||||
checksum = body.checksum.removeprefix("sha256:")
|
||||
if item.completed_at:
|
||||
if item.checksum_sha256 != checksum:
|
||||
raise DomainError("resource_state_conflict", 409, "Attachment checksum changed")
|
||||
return attachment_dto(item)
|
||||
try:
|
||||
head = await s3.head(item.storage_bucket, item.object_key)
|
||||
except Exception as exc:
|
||||
raise DomainError("dependency_unavailable", 503, "Object storage is unavailable") from exc
|
||||
if int(head["ContentLength"]) != item.size_bytes or head.get("ContentType") != item.mime_type:
|
||||
raise DomainError("attachment_checksum_mismatch", 400, "Uploaded metadata does not match")
|
||||
item.checksum_sha256 = checksum
|
||||
item.completed_at = datetime.now(UTC)
|
||||
session.add(audit("attachment.upload_completed", request_id, user.id, "attachment", item.id))
|
||||
await session.commit()
|
||||
return attachment_dto(item)
|
||||
|
||||
|
||||
def attachment_dto(item: MessageAttachment) -> dict[str, Any]:
|
||||
return {
|
||||
"attachment_id": item.id,
|
||||
"file_name": item.safe_file_name,
|
||||
"mime_type": item.mime_type,
|
||||
"size_bytes": item.size_bytes,
|
||||
"checksum": f"sha256:{item.checksum_sha256}" if item.checksum_sha256 else None,
|
||||
"scan_status": item.scan_status,
|
||||
"completed_at": item.completed_at,
|
||||
}
|
||||
|
||||
|
||||
async def send_message(
|
||||
session: AsyncSession,
|
||||
user: UserIdentity,
|
||||
dialog_id: uuid.UUID,
|
||||
body: MessageRequest,
|
||||
idem_key: str,
|
||||
request_id: str,
|
||||
settings: Settings,
|
||||
safety: SafetyClient,
|
||||
openlines: OpenLinesClient,
|
||||
s3: S3Client,
|
||||
fanout: RealtimeFanout,
|
||||
) -> dict[str, Any]:
|
||||
scope = f"dialogs.{dialog_id}.messages.create"
|
||||
fingerprint = hashlib.sha256(
|
||||
json.dumps(body.model_dump(mode="json"), sort_keys=True, separators=(",", ":")).encode()
|
||||
).hexdigest()
|
||||
idem = (
|
||||
await session.execute(
|
||||
select(IdempotencyRecord).where(
|
||||
IdempotencyRecord.scope == scope,
|
||||
IdempotencyRecord.user_id == user.id,
|
||||
IdempotencyRecord.idempotency_key == idem_key,
|
||||
)
|
||||
)
|
||||
).scalar_one_or_none()
|
||||
if idem:
|
||||
if idem.request_fingerprint != fingerprint:
|
||||
raise DomainError("idempotency_key_reused", 409, "Idempotency key was reused")
|
||||
if idem.status == "completed" and idem.response_body_json:
|
||||
if idem.response_status == 422:
|
||||
raise DomainError("message_blocked", 422, "Message was blocked by safety policy")
|
||||
return idem.response_body_json
|
||||
prior = (
|
||||
await session.execute(
|
||||
select(Message).where(
|
||||
Message.dialog_id == dialog_id,
|
||||
Message.client_idempotency_key == idem_key,
|
||||
)
|
||||
)
|
||||
).scalar_one_or_none()
|
||||
if prior and prior.delivery_status == "delivered":
|
||||
return message_dto(prior)
|
||||
raise DomainError(
|
||||
"dependency_unavailable",
|
||||
503,
|
||||
"Previous request is still being recovered",
|
||||
{"retry_after": 2},
|
||||
)
|
||||
dialog = await owned_dialog(session, user.id, dialog_id)
|
||||
now, message_id = datetime.now(UTC), uuid.uuid4()
|
||||
idem = IdempotencyRecord(
|
||||
scope=scope,
|
||||
user_id=user.id,
|
||||
idempotency_key=idem_key,
|
||||
request_fingerprint=fingerprint,
|
||||
status="in_progress",
|
||||
resource_type="message",
|
||||
resource_id=message_id,
|
||||
expires_at=now + timedelta(hours=24),
|
||||
)
|
||||
session.add(idem)
|
||||
attachment: MessageAttachment | None = None
|
||||
if isinstance(body, FileMessageRequest):
|
||||
attachment = (
|
||||
await session.execute(
|
||||
select(MessageAttachment).where(
|
||||
MessageAttachment.id == body.attachment_id,
|
||||
MessageAttachment.owner_user_id == user.id,
|
||||
MessageAttachment.dialog_id == dialog_id,
|
||||
MessageAttachment.record_status == "A",
|
||||
)
|
||||
)
|
||||
).scalar_one_or_none()
|
||||
if not attachment or not attachment.completed_at:
|
||||
raise DomainError("attachment_not_completed", 400, "Attachment upload is incomplete")
|
||||
if attachment.checksum_sha256 != body.checksum.removeprefix("sha256:"):
|
||||
raise DomainError("attachment_checksum_mismatch", 400, "Attachment checksum differs")
|
||||
text, kind = "", "file"
|
||||
else:
|
||||
text, kind = unicodedata.normalize("NFKC", body.text).strip(), "text"
|
||||
message = Message(
|
||||
id=message_id,
|
||||
dialog_id=dialog_id,
|
||||
sender_type="client",
|
||||
content_kind=kind,
|
||||
text=text,
|
||||
safety_status="pending",
|
||||
delivery_status="accepted",
|
||||
client_idempotency_key=idem_key,
|
||||
occurred_at=now,
|
||||
)
|
||||
session.add(message)
|
||||
if attachment:
|
||||
attachment.message_id = message_id
|
||||
session.add(audit("message.submitted", request_id, user.id, "message", message_id))
|
||||
await session.commit()
|
||||
await publish_message(fanout, message, settings, [attachment] if attachment else [])
|
||||
payload: dict[str, Any] = {"message_id": str(message_id), "content_kind": kind, "text": text}
|
||||
if attachment:
|
||||
payload["attachment"] = {
|
||||
"attachment_id": str(attachment.id),
|
||||
"quarantine_object_key": attachment.quarantine_object_key,
|
||||
"checksum": f"sha256:{attachment.checksum_sha256}",
|
||||
"mime_type": attachment.mime_type,
|
||||
"size_bytes": attachment.size_bytes,
|
||||
}
|
||||
try:
|
||||
verdict = await safety.check(payload, request_id)
|
||||
if verdict["_status"] == 203:
|
||||
task_id = verdict["task_id"]
|
||||
task = SafetyTask(
|
||||
task_id=task_id,
|
||||
message_id=message.id,
|
||||
attachment_id=attachment.id if attachment else None,
|
||||
quarantine_object_key=attachment.quarantine_object_key if attachment else None,
|
||||
status="polling",
|
||||
deadline_at=now
|
||||
+ timedelta(seconds=settings.message_safety_task_poll_max_sec + 900),
|
||||
next_poll_at=now,
|
||||
)
|
||||
session.add(task)
|
||||
await session.commit()
|
||||
deadline = time_monotonic() + settings.message_safety_task_poll_max_sec
|
||||
while time_monotonic() < deadline:
|
||||
await sleep(settings.message_safety_task_poll_interval_sec)
|
||||
verdict = await safety.poll(task_id, request_id)
|
||||
if verdict["_status"] != 203:
|
||||
break
|
||||
else:
|
||||
raise DependencyFailure(timeout=True)
|
||||
if verdict["_status"] == 403 or (
|
||||
verdict["_status"] == 400
|
||||
and verdict.get("error", {}).get("code") == "stub_final_error"
|
||||
and verdict.get("verdict") == "deny"
|
||||
):
|
||||
message.text = ""
|
||||
message.safety_status = "blocked"
|
||||
message.delivery_status = "rejected"
|
||||
if attachment and attachment.quarantine_object_key:
|
||||
attachment.scan_status = "infected"
|
||||
await s3.delete_quarantine(attachment.quarantine_object_key)
|
||||
session.add(audit("message.blocked", request_id, user.id, "message", message.id))
|
||||
idem.status = "completed"
|
||||
idem.response_status = 422
|
||||
idem.response_body_json = {
|
||||
"error": {"code": "message_blocked", "message_id": str(message.id)}
|
||||
}
|
||||
await session.commit()
|
||||
await publish_message_status(fanout, message, settings)
|
||||
raise DomainError("message_blocked", 422, "Message was blocked by safety policy")
|
||||
if verdict["_status"] != 200:
|
||||
raise DependencyFailure()
|
||||
if attachment and attachment.quarantine_object_key:
|
||||
destination = f"attachments/dialogs/{dialog_id}/{attachment.id}"
|
||||
await s3.promote(attachment.quarantine_object_key, destination)
|
||||
attachment.storage_bucket = settings.selectel_s3_bucket_attachments
|
||||
attachment.object_key = destination
|
||||
attachment.quarantine_object_key = None
|
||||
attachment.scan_status = "clean"
|
||||
message.safety_status = "allowed"
|
||||
outbox = DeliveryOutbox(
|
||||
message_id=message.id,
|
||||
external_chat_id=dialog_id,
|
||||
payload_json={
|
||||
"message_id": str(message.id),
|
||||
"external_chat_id": str(dialog_id),
|
||||
"occurred_at": message.occurred_at.isoformat(),
|
||||
"user": {"id": str(user.id), "display_name": user.phone_number},
|
||||
"message": {
|
||||
"content_kind": message.content_kind,
|
||||
"text": message.text,
|
||||
"files": (
|
||||
[
|
||||
{
|
||||
"attachment_id": str(attachment.id),
|
||||
"name": attachment.safe_file_name,
|
||||
"mime_type": attachment.mime_type,
|
||||
"size_bytes": attachment.size_bytes,
|
||||
"_storage_bucket": attachment.storage_bucket,
|
||||
"_object_key": attachment.object_key,
|
||||
}
|
||||
]
|
||||
if attachment
|
||||
else []
|
||||
),
|
||||
},
|
||||
},
|
||||
next_attempt_at=datetime.now(UTC),
|
||||
)
|
||||
session.add(outbox)
|
||||
await session.commit()
|
||||
await publish_message_status(fanout, message, settings)
|
||||
delivery_payload = await fresh_openlines_payload(outbox.payload_json, s3)
|
||||
await openlines.send(message.id, delivery_payload, request_id)
|
||||
message.delivery_status = "delivered"
|
||||
dialog.status = "waiting_for_company"
|
||||
dialog.last_message_at = datetime.now(UTC)
|
||||
outbox.status = "delivered"
|
||||
session.add(audit("message.delivered", request_id, user.id, "message", message.id))
|
||||
result = message_dto(message, [attachment] if attachment else [])
|
||||
idem.status = "completed"
|
||||
idem.response_status = 201
|
||||
idem.response_body_json = json.loads(json.dumps(result, default=str))
|
||||
await session.commit()
|
||||
await publish_message_status(fanout, message, settings)
|
||||
await publish_dialog_status(fanout, dialog)
|
||||
return result
|
||||
except DomainError:
|
||||
raise
|
||||
except DependencyFailure as exc:
|
||||
message.delivery_status = "failed"
|
||||
session.add(audit("message.failed", request_id, user.id, "message", message.id))
|
||||
await session.commit()
|
||||
await publish_message_status(fanout, message, settings)
|
||||
raise DomainError(
|
||||
"dependency_timeout" if exc.timeout else "dependency_unavailable",
|
||||
504 if exc.timeout else 503,
|
||||
"A required dependency did not complete the request",
|
||||
) from exc
|
||||
|
||||
|
||||
async def apply_inbox(
|
||||
session: AsyncSession,
|
||||
event: OpenLinesInbox,
|
||||
request_id: str,
|
||||
snapshot: SettingsSnapshot,
|
||||
s3: S3Client,
|
||||
http: httpx.AsyncClient,
|
||||
settings: Settings,
|
||||
fanout: RealtimeFanout,
|
||||
) -> tuple[dict[str, Any], int]:
|
||||
fingerprint = hashlib.sha256(event.model_dump_json().encode()).hexdigest()
|
||||
existing = (
|
||||
await session.execute(
|
||||
select(OpenLinesInboxReceipt).where(OpenLinesInboxReceipt.event_id == event.event_id)
|
||||
)
|
||||
).scalar_one_or_none()
|
||||
if existing:
|
||||
if existing.payload_fingerprint != fingerprint:
|
||||
raise DomainError("idempotency_key_reused", 409, "Event id was reused")
|
||||
return {"status": "duplicate"}, 200
|
||||
dialog = (
|
||||
await session.execute(
|
||||
select(Dialog).where(Dialog.id == event.external_chat_id, Dialog.record_status == "A")
|
||||
)
|
||||
).scalar_one_or_none()
|
||||
if not dialog:
|
||||
raise DomainError("not_found", 404, "Resource was not found")
|
||||
receipt = OpenLinesInboxReceipt(
|
||||
event_id=event.event_id,
|
||||
external_chat_id=event.external_chat_id,
|
||||
bitrix_message_id=event.bitrix_message_id,
|
||||
event_type=event.event_type,
|
||||
payload_fingerprint=fingerprint,
|
||||
status="processing",
|
||||
)
|
||||
session.add(receipt)
|
||||
message: Message | None = None
|
||||
attachment: MessageAttachment | None = None
|
||||
if event.event_type == "dialog.closed":
|
||||
dialog.status = "closed"
|
||||
dialog.closed_at = event.occurred_at
|
||||
else:
|
||||
assert event.message is not None
|
||||
inbound_file = event.message.files[0] if event.message.files else None
|
||||
attachment_data: tuple[str, int, str] | None = None
|
||||
if inbound_file:
|
||||
extension = PurePath(inbound_file.name).suffix.lower().lstrip(".")
|
||||
max_bytes = snapshot.integer("chat.attachments.max_size_mb") * 1024 * 1024
|
||||
if (
|
||||
extension not in snapshot.strings("chat.attachments.allowed_extensions")
|
||||
or inbound_file.mime_type
|
||||
not in snapshot.strings("chat.attachments.allowed_mime_types")
|
||||
or inbound_file.size_bytes > max_bytes
|
||||
):
|
||||
raise DomainError("validation_error", 400, "Inbound file is not allowed")
|
||||
attachment_id = uuid.uuid4()
|
||||
object_key = f"attachments/dialogs/{dialog.id}/{attachment_id}"
|
||||
try:
|
||||
actual_size, checksum = await s3.upload_inbound(
|
||||
http,
|
||||
str(inbound_file.download_url),
|
||||
object_key,
|
||||
inbound_file.mime_type,
|
||||
max_bytes,
|
||||
)
|
||||
except DependencyFailure as exc:
|
||||
raise DomainError(
|
||||
"dependency_unavailable", 503, "Inbound file transfer failed"
|
||||
) from exc
|
||||
if actual_size != inbound_file.size_bytes:
|
||||
raise DomainError("validation_error", 400, "Inbound file size differs")
|
||||
attachment_data = object_key, actual_size, checksum
|
||||
message = Message(
|
||||
dialog_id=dialog.id,
|
||||
sender_type="company",
|
||||
content_kind="file" if inbound_file else "text",
|
||||
text=event.message.text,
|
||||
safety_status="allowed",
|
||||
delivery_status="delivered",
|
||||
external_message_id=event.bitrix_message_id,
|
||||
occurred_at=event.occurred_at,
|
||||
)
|
||||
session.add(message)
|
||||
await session.flush()
|
||||
if inbound_file and attachment_data:
|
||||
object_key, actual_size, checksum = attachment_data
|
||||
safe_name = re.sub(
|
||||
r"[^A-Za-z0-9._-]",
|
||||
"_",
|
||||
unicodedata.normalize("NFKC", inbound_file.name),
|
||||
)
|
||||
attachment = MessageAttachment(
|
||||
dialog_id=dialog.id,
|
||||
message_id=message.id,
|
||||
owner_user_id=dialog.user_id,
|
||||
direction="company_inbound",
|
||||
original_file_name=inbound_file.name,
|
||||
safe_file_name=safe_name,
|
||||
mime_type=inbound_file.mime_type,
|
||||
size_bytes=actual_size,
|
||||
checksum_sha256=checksum,
|
||||
scan_status="clean",
|
||||
storage_bucket=s3.settings.selectel_s3_bucket_attachments,
|
||||
object_key=object_key,
|
||||
completed_at=datetime.now(UTC),
|
||||
)
|
||||
session.add(attachment)
|
||||
receipt.message_id = message.id
|
||||
dialog.status = "waiting_for_client"
|
||||
dialog.last_message_at = event.occurred_at
|
||||
receipt.status = "applied"
|
||||
session.add(audit("openlines.inbox_applied", request_id, dialog.user_id, "dialog", dialog.id))
|
||||
await session.commit()
|
||||
if message is not None:
|
||||
await publish_message(fanout, message, settings, [attachment] if attachment else [])
|
||||
await publish_dialog_status(fanout, dialog)
|
||||
return {"status": "applied"}, 201
|
||||
|
||||
|
||||
async def sleep(seconds: float) -> None:
|
||||
import asyncio
|
||||
|
||||
await asyncio.sleep(seconds)
|
||||
|
||||
|
||||
def time_monotonic() -> float:
|
||||
import time
|
||||
|
||||
return time.monotonic()
|
||||
@@ -0,0 +1,78 @@
|
||||
from functools import lru_cache
|
||||
|
||||
from pydantic import AnyHttpUrl, Field, SecretStr
|
||||
from pydantic_settings import BaseSettings, SettingsConfigDict
|
||||
|
||||
|
||||
class Settings(BaseSettings):
|
||||
model_config = SettingsConfigDict(
|
||||
env_file=".env", env_file_encoding="utf-8", case_sensitive=True, extra="ignore"
|
||||
)
|
||||
|
||||
app_env: str = Field(alias="APP_ENV")
|
||||
api_port: int = Field(default=8000, alias="API_PORT")
|
||||
log_level: str = Field(default="INFO", alias="LOG_LEVEL")
|
||||
database_url: str = Field(alias="DATABASE_URL")
|
||||
redis_url: str = Field(alias="REDIS_URL")
|
||||
redis_realtime_url: str = Field(alias="REDIS_REALTIME_URL")
|
||||
|
||||
keycloak_public_url: AnyHttpUrl = Field(alias="KEYCLOAK_PUBLIC_URL")
|
||||
keycloak_internal_url: AnyHttpUrl = Field(alias="KEYCLOAK_INTERNAL_URL")
|
||||
keycloak_realm: str = Field(alias="KEYCLOAK_REALM")
|
||||
keycloak_audience: str = Field(alias="KEYCLOAK_AUDIENCE")
|
||||
jwks_cache_ttl_seconds: int = 300
|
||||
jwks_stale_grace_seconds: int = 900
|
||||
|
||||
message_safety_url: AnyHttpUrl = Field(alias="MESSAGE_SAFETY_URL")
|
||||
message_safety_service_token: SecretStr = Field(alias="MESSAGE_SAFETY_SERVICE_TOKEN")
|
||||
message_safety_post_timeout_sec: float = Field(
|
||||
default=5, alias="MESSAGE_SAFETY_POST_TIMEOUT_SEC"
|
||||
)
|
||||
message_safety_task_poll_interval_sec: float = Field(
|
||||
default=2, alias="MESSAGE_SAFETY_TASK_POLL_INTERVAL_SEC"
|
||||
)
|
||||
message_safety_task_poll_max_sec: float = Field(
|
||||
default=300, alias="MESSAGE_SAFETY_TASK_POLL_MAX_SEC"
|
||||
)
|
||||
message_safety_circuit_failure_threshold: int = Field(
|
||||
default=5, alias="MESSAGE_SAFETY_CIRCUIT_FAILURE_THRESHOLD"
|
||||
)
|
||||
message_safety_circuit_open_sec: int = Field(
|
||||
default=30, alias="MESSAGE_SAFETY_CIRCUIT_OPEN_SEC"
|
||||
)
|
||||
|
||||
bitrix_local_app_base_url: AnyHttpUrl = Field(alias="BITRIX_LOCAL_APP_BASE_URL")
|
||||
bitrix_local_app_internal_token: SecretStr = Field(alias="BITRIX_LOCAL_APP_INTERNAL_TOKEN")
|
||||
bitrix_api_inbox_token: SecretStr = Field(alias="BITRIX_API_INBOX_TOKEN")
|
||||
bitrix_local_app_http_timeout_sec: float = Field(
|
||||
default=10, alias="BITRIX_LOCAL_APP_HTTP_TIMEOUT_SEC"
|
||||
)
|
||||
bitrix_local_app_circuit_failure_threshold: int = Field(
|
||||
default=5, alias="BITRIX_LOCAL_APP_CIRCUIT_FAILURE_THRESHOLD"
|
||||
)
|
||||
bitrix_local_app_circuit_open_sec: int = Field(
|
||||
default=30, alias="BITRIX_LOCAL_APP_CIRCUIT_OPEN_SEC"
|
||||
)
|
||||
keycloak_settings_bridge_token: SecretStr = Field(alias="KEYCLOAK_SETTINGS_BRIDGE_TOKEN")
|
||||
|
||||
selectel_s3_endpoint_url: AnyHttpUrl = Field(alias="SELECTEL_S3_ENDPOINT_URL")
|
||||
selectel_s3_bucket_documents: str = Field(alias="SELECTEL_S3_BUCKET_DOCUMENTS")
|
||||
selectel_s3_bucket_attachments: str = Field(alias="SELECTEL_S3_BUCKET_ATTACHMENTS")
|
||||
selectel_s3_bucket_quarantine: str = Field(alias="SELECTEL_S3_BUCKET_QUARANTINE")
|
||||
selectel_s3_access_key: SecretStr = Field(alias="SELECTEL_S3_ACCESS_KEY")
|
||||
selectel_s3_secret_key: SecretStr = Field(alias="SELECTEL_S3_SECRET_KEY")
|
||||
otel_exporter_otlp_endpoint: str | None = Field(
|
||||
default=None, alias="OTEL_EXPORTER_OTLP_ENDPOINT"
|
||||
)
|
||||
cursor_hmac_secret: SecretStr = Field(alias="CURSOR_HMAC_SECRET")
|
||||
trusted_proxy_cidrs: str = Field(default="127.0.0.1/32", alias="TRUSTED_PROXY_CIDRS")
|
||||
worker_poll_interval_sec: float = Field(default=2, alias="WORKER_POLL_INTERVAL_SEC")
|
||||
|
||||
@property
|
||||
def issuer(self) -> str:
|
||||
return f"{str(self.keycloak_public_url).rstrip('/')}/realms/{self.keycloak_realm}"
|
||||
|
||||
|
||||
@lru_cache
|
||||
def get_settings() -> Settings:
|
||||
return Settings()
|
||||
@@ -0,0 +1,234 @@
|
||||
import asyncio
|
||||
import uuid
|
||||
from datetime import UTC, datetime, timedelta
|
||||
|
||||
import httpx
|
||||
import redis.asyncio as redis
|
||||
import structlog
|
||||
from sqlalchemy import select
|
||||
|
||||
from app.db import Database, DeliveryOutbox, Dialog, Message, MessageAttachment, SafetyTask
|
||||
from app.integrations import (
|
||||
DependencyFailure,
|
||||
OpenLinesClient,
|
||||
S3Client,
|
||||
SafetyClient,
|
||||
fresh_openlines_payload,
|
||||
)
|
||||
from app.realtime import RealtimeFanout
|
||||
from app.services import publish_dialog_status, publish_message_status
|
||||
from app.settings import Settings, get_settings
|
||||
|
||||
log = structlog.get_logger()
|
||||
|
||||
|
||||
async def delivery_once(
|
||||
db: Database,
|
||||
client: OpenLinesClient,
|
||||
s3: S3Client,
|
||||
fanout: RealtimeFanout,
|
||||
settings: Settings,
|
||||
worker_id: str,
|
||||
batch_size: int = 20,
|
||||
) -> int:
|
||||
async with db.sessions() as session:
|
||||
rows = (
|
||||
(
|
||||
await session.execute(
|
||||
select(DeliveryOutbox)
|
||||
.where(
|
||||
DeliveryOutbox.status.in_(["pending", "retry"]),
|
||||
DeliveryOutbox.next_attempt_at <= datetime.now(UTC),
|
||||
)
|
||||
.with_for_update(skip_locked=True)
|
||||
.limit(batch_size)
|
||||
)
|
||||
)
|
||||
.scalars()
|
||||
.all()
|
||||
)
|
||||
ids = [row.id for row in rows]
|
||||
for row in rows:
|
||||
row.status = "processing"
|
||||
row.locked_at = datetime.now(UTC)
|
||||
row.locked_by = worker_id
|
||||
await session.commit()
|
||||
for row_id in ids:
|
||||
async with db.sessions() as session:
|
||||
row = await session.get(DeliveryOutbox, row_id, with_for_update=True)
|
||||
if row is None:
|
||||
continue
|
||||
message = None
|
||||
dialog = None
|
||||
try:
|
||||
payload = await fresh_openlines_payload(row.payload_json, s3)
|
||||
await client.send(row.message_id, payload, f"worker-{worker_id}")
|
||||
row.status = "delivered"
|
||||
message = await session.get(Message, row.message_id)
|
||||
if message:
|
||||
message.delivery_status = "delivered"
|
||||
dialog = await session.get(Dialog, message.dialog_id)
|
||||
if dialog:
|
||||
dialog.status = "waiting_for_company"
|
||||
dialog.last_message_at = datetime.now(UTC)
|
||||
except DependencyFailure:
|
||||
row.attempt_count += 1
|
||||
row.status = "dead_letter" if row.attempt_count >= 12 else "retry"
|
||||
row.next_attempt_at = datetime.now(UTC) + timedelta(
|
||||
seconds=min(3600, 2**row.attempt_count)
|
||||
)
|
||||
row.last_error_code = "dependency_unavailable"
|
||||
row.locked_at = None
|
||||
row.locked_by = None
|
||||
await session.commit()
|
||||
if message:
|
||||
await publish_message_status(fanout, message, settings)
|
||||
if dialog:
|
||||
await publish_dialog_status(fanout, dialog)
|
||||
return len(ids)
|
||||
|
||||
|
||||
async def safety_once(
|
||||
db: Database,
|
||||
safety: SafetyClient,
|
||||
s3: S3Client,
|
||||
fanout: RealtimeFanout,
|
||||
settings: Settings,
|
||||
worker_id: str,
|
||||
batch_size: int = 20,
|
||||
) -> int:
|
||||
async with db.sessions() as session:
|
||||
rows = (
|
||||
(
|
||||
await session.execute(
|
||||
select(SafetyTask)
|
||||
.where(
|
||||
SafetyTask.status.in_(["polling", "failed"]),
|
||||
SafetyTask.next_poll_at <= datetime.now(UTC),
|
||||
SafetyTask.deadline_at > datetime.now(UTC),
|
||||
)
|
||||
.with_for_update(skip_locked=True)
|
||||
.limit(batch_size)
|
||||
)
|
||||
)
|
||||
.scalars()
|
||||
.all()
|
||||
)
|
||||
ids = [row.id for row in rows]
|
||||
for row in rows:
|
||||
row.locked_at, row.locked_by = datetime.now(UTC), worker_id
|
||||
await session.commit()
|
||||
for task_id in ids:
|
||||
async with db.sessions() as session:
|
||||
task = await session.get(SafetyTask, task_id, with_for_update=True)
|
||||
if task is None:
|
||||
continue
|
||||
message = None
|
||||
try:
|
||||
verdict = await safety.poll(task.task_id, f"worker-{worker_id}")
|
||||
message = await session.get(Message, task.message_id)
|
||||
attachment = (
|
||||
await session.get(MessageAttachment, task.attachment_id)
|
||||
if task.attachment_id
|
||||
else None
|
||||
)
|
||||
if verdict["_status"] == 200 and message:
|
||||
if attachment and attachment.quarantine_object_key:
|
||||
destination = f"attachments/dialogs/{message.dialog_id}/{attachment.id}"
|
||||
await s3.promote(attachment.quarantine_object_key, destination)
|
||||
attachment.storage_bucket = s3.settings.selectel_s3_bucket_attachments
|
||||
attachment.object_key = destination
|
||||
attachment.quarantine_object_key = None
|
||||
attachment.scan_status = "clean"
|
||||
message.safety_status = "allowed"
|
||||
task.status = "completed"
|
||||
elif verdict["_status"] == 403 and message:
|
||||
message.text = ""
|
||||
message.safety_status = "blocked"
|
||||
message.delivery_status = "rejected"
|
||||
if attachment and attachment.quarantine_object_key:
|
||||
await s3.delete_quarantine(attachment.quarantine_object_key)
|
||||
attachment.scan_status = "infected"
|
||||
task.status = "completed"
|
||||
else:
|
||||
task.next_poll_at = datetime.now(UTC) + timedelta(seconds=2)
|
||||
except DependencyFailure:
|
||||
task.attempt_count += 1
|
||||
task.status = "failed"
|
||||
task.next_poll_at = datetime.now(UTC) + timedelta(
|
||||
seconds=min(300, 2**task.attempt_count)
|
||||
)
|
||||
task.locked_at = None
|
||||
task.locked_by = None
|
||||
await session.commit()
|
||||
if message:
|
||||
await publish_message_status(fanout, message, settings)
|
||||
return len(ids)
|
||||
|
||||
|
||||
async def cleanup_once(db: Database, s3: S3Client, batch_size: int = 100) -> int:
|
||||
async with db.sessions() as session:
|
||||
rows = (
|
||||
(
|
||||
await session.execute(
|
||||
select(MessageAttachment)
|
||||
.where(
|
||||
MessageAttachment.record_status == "A",
|
||||
MessageAttachment.quarantine_object_key.is_not(None),
|
||||
MessageAttachment.upload_expires_at < datetime.now(UTC),
|
||||
MessageAttachment.message_id.is_(None),
|
||||
)
|
||||
.with_for_update(skip_locked=True)
|
||||
.limit(batch_size)
|
||||
)
|
||||
)
|
||||
.scalars()
|
||||
.all()
|
||||
)
|
||||
for row in rows:
|
||||
if row.quarantine_object_key:
|
||||
await s3.delete_quarantine(row.quarantine_object_key)
|
||||
row.record_status = "D"
|
||||
row.status_changed_at = datetime.now(UTC)
|
||||
row.status_change_reason = "expired_quarantine_cleanup"
|
||||
await session.commit()
|
||||
return len(rows)
|
||||
|
||||
|
||||
async def loop(kind: str) -> None:
|
||||
settings = get_settings()
|
||||
db = Database(settings.database_url)
|
||||
http = httpx.AsyncClient()
|
||||
safety = SafetyClient(settings, http)
|
||||
openlines = OpenLinesClient(settings, http)
|
||||
s3 = S3Client(settings)
|
||||
redis_rt = redis.from_url(settings.redis_realtime_url, decode_responses=True)
|
||||
fanout = RealtimeFanout(redis_rt)
|
||||
worker_id = f"{kind}-{uuid.uuid4()}"
|
||||
try:
|
||||
while True:
|
||||
count = 0
|
||||
if kind == "delivery":
|
||||
count = await delivery_once(db, openlines, s3, fanout, settings, worker_id)
|
||||
elif kind == "safety":
|
||||
count = await safety_once(db, safety, s3, fanout, settings, worker_id)
|
||||
else:
|
||||
count = await cleanup_once(db, s3)
|
||||
if not count:
|
||||
await asyncio.sleep(settings.worker_poll_interval_sec)
|
||||
finally:
|
||||
await http.aclose()
|
||||
await redis_rt.aclose()
|
||||
await db.close()
|
||||
|
||||
|
||||
def delivery_main() -> None:
|
||||
asyncio.run(loop("delivery"))
|
||||
|
||||
|
||||
def safety_main() -> None:
|
||||
asyncio.run(loop("safety"))
|
||||
|
||||
|
||||
def cleanup_main() -> None:
|
||||
asyncio.run(loop("cleanup"))
|
||||
Reference in New Issue
Block a user