Разработана первая версия приложений

This commit is contained in:
mi
2026-07-10 18:06:14 +03:00
parent aa8761d1b3
commit 8c7b4074c4
162 changed files with 12178 additions and 16 deletions
@@ -0,0 +1,8 @@
.env
.git
.mypy_cache
.pytest_cache
.ruff_cache
__pycache__
*.py[cod]
tests
+19
View File
@@ -0,0 +1,19 @@
FROM python:3.12-slim AS builder
WORKDIR /build
RUN pip install --no-cache-dir --upgrade pip build
COPY pyproject.toml ./
COPY app ./app
RUN python -m build --wheel
FROM python:3.12-slim
ENV PYTHONDONTWRITEBYTECODE=1 PYTHONUNBUFFERED=1
RUN addgroup --system --gid 10001 han && adduser --system --uid 10001 --ingroup han han
WORKDIR /app
COPY --from=builder /build/dist/*.whl /tmp/
RUN pip install --no-cache-dir /tmp/*.whl && rm -f /tmp/*.whl
COPY alembic.ini ./
COPY alembic ./alembic
USER 10001:10001
EXPOSE 8000
ENTRYPOINT ["uvicorn"]
CMD ["app.main:app", "--host", "0.0.0.0", "--port", "8000", "--no-proxy-headers"]
+68
View File
@@ -0,0 +1,68 @@
# HAN Chat API backend
FastAPI-сервис публичного API, internal Open Lines/settings API, WebSocket realtime,
Message Safety orchestration, S3 lifecycle и PostgreSQL workers.
## Запуск
Python 3.12+:
```bash
python -m venv .venv
.venv/Scripts/pip install -e ".[dev]"
alembic upgrade head
uvicorn app.main:app --host 0.0.0.0 --port 8000
```
Миграции выполняются отдельным deployment step. Приложение не применяет DDL при старте.
Workers запускаются независимо:
```bash
han-delivery-worker
han-safety-worker
han-cleanup-worker
```
## Переменные окружения
Сервис читает только инфраструктурные параметры и секреты из корневого `backend/.env`.
Локальный `.env` не коммитится. Business settings создаются миграцией в
`han_app.app_settings`.
Обязательны:
- `APP_ENV`, `API_PORT`, `LOG_LEVEL`, `DATABASE_URL`;
- `REDIS_URL`, `REDIS_REALTIME_URL`;
- `KEYCLOAK_PUBLIC_URL`, `KEYCLOAK_INTERNAL_URL`, `KEYCLOAK_REALM`,
`KEYCLOAK_AUDIENCE`;
- `MESSAGE_SAFETY_URL`, `MESSAGE_SAFETY_SERVICE_TOKEN`,
`MESSAGE_SAFETY_POST_TIMEOUT_SEC`, `MESSAGE_SAFETY_TASK_POLL_INTERVAL_SEC`,
`MESSAGE_SAFETY_TASK_POLL_MAX_SEC`,
`MESSAGE_SAFETY_CIRCUIT_FAILURE_THRESHOLD`, `MESSAGE_SAFETY_CIRCUIT_OPEN_SEC`;
- `BITRIX_LOCAL_APP_BASE_URL`, `BITRIX_LOCAL_APP_INTERNAL_TOKEN`,
`BITRIX_API_INBOX_TOKEN`, `BITRIX_LOCAL_APP_HTTP_TIMEOUT_SEC`,
`BITRIX_LOCAL_APP_CIRCUIT_FAILURE_THRESHOLD`,
`BITRIX_LOCAL_APP_CIRCUIT_OPEN_SEC`;
- `KEYCLOAK_SETTINGS_BRIDGE_TOKEN`;
- `SELECTEL_S3_ENDPOINT_URL`, `SELECTEL_S3_BUCKET_DOCUMENTS`,
`SELECTEL_S3_BUCKET_ATTACHMENTS`, `SELECTEL_S3_BUCKET_QUARANTINE`,
`SELECTEL_S3_ACCESS_KEY`, `SELECTEL_S3_SECRET_KEY`;
- `CURSOR_HMAC_SECRET` — случайный секрет не короче 32 байт;
- `OTEL_EXPORTER_OTLP_ENDPOINT` — опциональный endpoint collector.
Токены генерируются `openssl rand -hex 32`. S3 read-only credentials Message Safety
не передаются этому контейнеру. В production подключение PostgreSQL должно использовать
TLS, а internal endpoints — быть доступны только из backend-сети.
## Проверки
```bash
ruff check .
ruff format --check .
mypy app
pytest
```
`/health/live` проверяет процесс. `/health/ready` проверяет критические зависимости и
возвращает `503`, если сервис не может безопасно обслуживать protected API.
+37
View File
@@ -0,0 +1,37 @@
[alembic]
script_location = alembic
prepend_sys_path = .
path_separator = os
[loggers]
keys = root,sqlalchemy,alembic
[handlers]
keys = console
[formatters]
keys = generic
[logger_root]
level = WARN
handlers = console
[logger_sqlalchemy]
level = WARN
handlers =
qualname = sqlalchemy.engine
[logger_alembic]
level = INFO
handlers =
qualname = alembic
[handler_console]
class = StreamHandler
args = (sys.stderr,)
level = NOTSET
formatter = generic
[formatter_generic]
format = %(levelname)-5.5s [%(name)s] %(message)s
datefmt = %H:%M:%S
@@ -0,0 +1,52 @@
import asyncio
from logging.config import fileConfig
from sqlalchemy import pool
from sqlalchemy.ext.asyncio import async_engine_from_config
from alembic import context
from app.db import Base
from app.settings import get_settings
config = context.config
if config.config_file_name:
fileConfig(config.config_file_name)
config.set_main_option("sqlalchemy.url", get_settings().database_url)
target_metadata = Base.metadata
def do_run_migrations(connection) -> None:
context.configure(
connection=connection,
target_metadata=target_metadata,
include_schemas=True,
version_table_schema="han_app",
compare_type=True,
)
with context.begin_transaction():
context.run_migrations()
async def run_async_migrations() -> None:
connectable = async_engine_from_config(
config.get_section(config.config_ini_section, {}),
prefix="sqlalchemy.",
poolclass=pool.NullPool,
)
async with connectable.connect() as connection:
await connection.run_sync(do_run_migrations)
await connectable.dispose()
if context.is_offline_mode():
context.configure(
url=config.get_main_option("sqlalchemy.url"),
target_metadata=target_metadata,
literal_binds=True,
include_schemas=True,
version_table_schema="han_app",
)
with context.begin_transaction():
context.run_migrations()
else:
asyncio.run(run_async_migrations())
@@ -0,0 +1,153 @@
"""Initial han_app schema, seed and CRM sync triggers.
Revision ID: 0001_initial
Revises:
Create Date: 2026-07-10
"""
from collections.abc import Sequence
from alembic import op
from app.db import Base
revision: str = "0001_initial"
down_revision: str | None = None
branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None
SEED = {
"auth.phone.enabled": ("true", "boolean", True),
"auth.password.enabled": ("false", "boolean", True),
"otp.phone.max_send_attempts_per_24h": ("3", "integer", False),
"otp.phone.min_seconds_between_attempts": ("30", "integer", False),
"operator.call.phone": ("+74999591007", "string", True),
"consent.personal_data.required": ("true", "boolean", True),
"consent.personal_data.document_url": (
"https://www.han0107.ru/privacy/persdata-agree-mobile",
"string",
True,
),
"consent.personal_data.version": ("2026-06-10", "string", True),
"consent.user_agreement.required": ("true", "boolean", True),
"consent.user_agreement.document_url": (
"https://www.han0107.ru/user-agreement",
"string",
True,
),
"consent.user_agreement.version": ("2026-06-10", "string", True),
"consent.marketing.required": ("false", "boolean", True),
"consent.marketing.version": ("2026-06-10", "string", True),
"chat.attachments.allowed_extensions": (
"jpg,jpeg,png,webp,heic,heif,pdf",
"string_list",
True,
),
"chat.attachments.allowed_mime_types": (
"image/jpeg,image/png,image/webp,image/heic,image/heif,application/pdf",
"string_list",
True,
),
"chat.attachments.disallowed_extensions": ("svg,doc,docx,xls,xlsx,csv", "string_list", False),
"chat.attachments.max_size_mb": ("5", "integer", True),
"chat.attachments.storage": ("selectel_s3", "string", False),
"chat.attachments.upload_mode": ("presigned_put", "string", False),
"chat.attachments.safety_scan_required": ("true", "boolean", False),
"chat.attachments.presigned_upload_ttl_seconds": ("600", "integer", False),
"rate_limit.message_send.per_user": ("30/minute", "string", False),
"rate_limit.message_send.per_dialog": ("20/minute", "string", False),
"rate_limit.download_url.per_user": ("60/hour", "string", False),
"rate_limit.public_endpoints.per_ip": ("60/minute", "string", False),
"rate_limit.login.per_ip": ("10/minute", "string", False),
"ux.session.idle_timeout_minutes": ("30", "integer", True),
"security.cors.allowed_origins": ("https://tohin.ru", "string_list", False),
"security.public_cache.max_age_seconds": ("3600", "integer", False),
}
def upgrade() -> None:
bind = op.get_bind()
op.execute("CREATE EXTENSION IF NOT EXISTS pgcrypto")
op.execute("CREATE SCHEMA IF NOT EXISTS han_app")
Base.metadata.create_all(bind=bind, checkfirst=True)
op.execute(
"""
CREATE UNIQUE INDEX IF NOT EXISTS uq_dialog_one_active_per_user
ON han_app.dialogs(user_id)
WHERE record_status='A'
AND status IN ('open','waiting_for_company','waiting_for_client');
CREATE UNIQUE INDEX IF NOT EXISTS uq_profiles_active_bitrix_contact
ON han_app.client_profiles(bitrix_contact_id)
WHERE bitrix_contact_id IS NOT NULL AND record_status='A';
CREATE OR REPLACE FUNCTION han_app.enqueue_contact_sync()
RETURNS trigger
LANGUAGE plpgsql
SECURITY INVOKER
SET search_path = han_app, pg_temp
AS $$
DECLARE
v_entity_id uuid;
v_task_type text;
v_dedup text;
BEGIN
IF current_setting('han.sync_suppress', true) = 'true' THEN
RETURN NEW;
END IF;
IF TG_TABLE_NAME = 'user_identities' THEN
v_entity_id := NEW.id;
v_task_type := CASE WHEN TG_OP = 'INSERT'
THEN 'contact.map_or_create' ELSE 'contact.update' END;
ELSE
v_entity_id := NEW.user_id;
v_task_type := CASE WHEN TG_OP = 'INSERT'
THEN 'contact.map_or_create' ELSE 'contact.update' END;
END IF;
v_dedup := v_task_type || ':' || v_entity_id::text || ':' ||
encode(digest(row_to_json(NEW)::text, 'sha256'), 'hex');
INSERT INTO han_app.sync_queue
(id, task_type, entity_type, entity_id, dedup_key, payload_json,
status, attempt_count, next_attempt_at, created_at, updated_at)
VALUES
(gen_random_uuid(), v_task_type, 'contact', v_entity_id, v_dedup,
jsonb_build_object('entity_id', v_entity_id), 'pending', 0,
now(), now(), now())
ON CONFLICT (dedup_key) DO NOTHING;
RETURN NEW;
END;
$$;
DROP TRIGGER IF EXISTS trg_identity_contact_sync ON han_app.user_identities;
CREATE TRIGGER trg_identity_contact_sync
AFTER INSERT OR UPDATE OF phone_number, record_status
ON han_app.user_identities
FOR EACH ROW EXECUTE FUNCTION han_app.enqueue_contact_sync();
DROP TRIGGER IF EXISTS trg_profile_contact_sync ON han_app.client_profiles;
CREATE TRIGGER trg_profile_contact_sync
AFTER INSERT OR UPDATE OF full_name, citizenship, russian_phone,
foreign_phone, email, record_status
ON han_app.client_profiles
FOR EACH ROW EXECUTE FUNCTION han_app.enqueue_contact_sync();
"""
)
for key, (value, value_type, public) in SEED.items():
bind.exec_driver_sql(
"""
INSERT INTO han_app.app_settings
(setting_key, setting_value, value_type, is_public, record_status, updated_at)
VALUES (%s, %s, %s, %s, 'A', now())
ON CONFLICT (setting_key) DO UPDATE SET
setting_value = EXCLUDED.setting_value,
value_type = EXCLUDED.value_type,
is_public = EXCLUDED.is_public,
record_status = 'A',
updated_at = now()
""",
(key, value, value_type, public),
)
def downgrade() -> None:
raise RuntimeError("Initial data migration is forward-only")
@@ -0,0 +1 @@
"""HAN Chat API backend."""
+106
View File
@@ -0,0 +1,106 @@
import asyncio
import time
from dataclasses import dataclass
from typing import Any
import httpx
import jwt
import phonenumbers
from jwt import ExpiredSignatureError, InvalidTokenError, PyJWK
from app.settings import Settings
class AuthError(Exception):
def __init__(self, code: str = "unauthorized") -> None:
self.code = code
@dataclass(frozen=True, slots=True)
class Principal:
subject: str
phone_number: str | None
claims: dict[str, Any]
class JWKSValidator:
def __init__(self, settings: Settings, http: httpx.AsyncClient) -> None:
self.settings = settings
self.http = http
self._keys: dict[str, PyJWK] = {}
self._loaded_at = 0.0
self._lock = asyncio.Lock()
async def refresh(self) -> None:
async with self._lock:
discovery_url = (
f"{str(self.settings.keycloak_internal_url).rstrip('/')}"
f"/realms/{self.settings.keycloak_realm}/.well-known/openid-configuration"
)
discovery = (await self.http.get(discovery_url, timeout=3)).raise_for_status().json()
jwks_uri = discovery["jwks_uri"]
public_prefix = str(self.settings.keycloak_public_url).rstrip("/")
internal_prefix = str(self.settings.keycloak_internal_url).rstrip("/")
if jwks_uri.startswith(public_prefix):
jwks_uri = internal_prefix + jwks_uri[len(public_prefix) :]
payload = (await self.http.get(jwks_uri, timeout=3)).raise_for_status().json()
self._keys = {
key["kid"]: PyJWK.from_dict(key)
for key in payload.get("keys", [])
if key.get("kid") and key.get("kty") == "RSA"
}
self._loaded_at = time.monotonic()
async def validate(self, token: str) -> Principal:
try:
header = jwt.get_unverified_header(token)
except InvalidTokenError as exc:
raise AuthError() from exc
if header.get("alg") != "RS256" or not header.get("kid"):
raise AuthError()
kid = str(header["kid"])
stale = time.monotonic() - self._loaded_at > self.settings.jwks_cache_ttl_seconds
if stale or kid not in self._keys:
try:
await self.refresh()
except (httpx.HTTPError, KeyError, ValueError):
if (
kid not in self._keys
or time.monotonic() - self._loaded_at > self.settings.jwks_stale_grace_seconds
):
raise AuthError() from None
key = self._keys.get(kid)
if key is None:
raise AuthError()
try:
claims = jwt.decode(
token,
key.key,
algorithms=["RS256"],
audience=self.settings.keycloak_audience,
issuer=self.settings.issuer,
options={"require": ["exp", "sub"]},
leeway=30,
)
except ExpiredSignatureError as exc:
raise AuthError("token_expired") from exc
except InvalidTokenError as exc:
raise AuthError() from exc
subject = claims.get("sub")
if not isinstance(subject, str) or not subject:
raise AuthError()
return Principal(subject, canonical_phone(claims), claims)
def canonical_phone(claims: dict[str, Any]) -> str | None:
for name in ("phone_number", "preferred_username"):
value = claims.get(name)
if not isinstance(value, str):
continue
try:
number = phonenumbers.parse(value, None)
except phonenumbers.NumberParseException:
continue
if phonenumbers.is_valid_number(number) and value.startswith("+"):
return phonenumbers.format_number(number, phonenumbers.PhoneNumberFormat.E164)
return None
@@ -0,0 +1 @@
"""Operational command-line entry points."""
@@ -0,0 +1,115 @@
from __future__ import annotations
import argparse
import asyncio
from pathlib import Path
from typing import Any
import yaml
from sqlalchemy import func, or_
from sqlalchemy.dialects.postgresql import insert
from app.db import AppSetting, Database
from app.settings import get_settings
VALUE_TYPES = {"boolean", "integer", "string", "string_list"}
def load_seed(path: Path) -> list[dict[str, Any]]:
document = yaml.safe_load(path.read_text(encoding="utf-8"))
if not isinstance(document, dict) or document.get("schema_version") != 1:
raise ValueError("settings file must have schema_version: 1")
settings = document.get("settings")
if not isinstance(settings, dict) or not settings:
raise ValueError("settings file must contain a non-empty settings mapping")
rows: list[dict[str, Any]] = []
for key, raw in settings.items():
if not isinstance(key, str) or not key.strip():
raise ValueError("setting keys must be non-empty strings")
if not isinstance(raw, dict):
raise ValueError(f"{key}: setting must be a mapping")
value_type = raw.get("type")
if value_type not in VALUE_TYPES:
raise ValueError(f"{key}: unsupported type {value_type!r}")
if not isinstance(raw.get("public"), bool):
raise ValueError(f"{key}: public must be a boolean")
description = raw.get("description")
if description is not None and not isinstance(description, str):
raise ValueError(f"{key}: description must be a string")
rows.append(
{
"setting_key": key,
"setting_value": serialize_value(key, value_type, raw.get("value")),
"value_type": value_type,
"is_public": raw["public"],
"description": description,
"record_status": "A",
}
)
return rows
def serialize_value(key: str, value_type: str, value: Any) -> str:
if value_type == "boolean":
if not isinstance(value, bool):
raise ValueError(f"{key}: boolean value expected")
return str(value).lower()
if value_type == "integer":
if not isinstance(value, int) or isinstance(value, bool):
raise ValueError(f"{key}: integer value expected")
return str(value)
if value_type == "string_list":
if isinstance(value, list) and all(isinstance(item, str) for item in value):
return ",".join(value)
if isinstance(value, str):
return value
raise ValueError(f"{key}: string or list of strings expected")
if not isinstance(value, str):
raise ValueError(f"{key}: string value expected")
return value
async def seed(path: Path) -> int:
rows = load_seed(path)
database = Database(get_settings().database_url)
try:
async with database.sessions() as session:
for row in rows:
statement = insert(AppSetting).values(**row)
excluded = statement.excluded
statement = statement.on_conflict_do_update(
index_elements=[AppSetting.setting_key],
set_={
"setting_value": excluded.setting_value,
"value_type": excluded.value_type,
"is_public": excluded.is_public,
"description": excluded.description,
"record_status": "A",
"updated_at": func.now(),
},
where=or_(
AppSetting.setting_value.is_distinct_from(excluded.setting_value),
AppSetting.value_type.is_distinct_from(excluded.value_type),
AppSetting.is_public.is_distinct_from(excluded.is_public),
AppSetting.description.is_distinct_from(excluded.description),
AppSetting.record_status != "A",
),
)
await session.execute(statement)
await session.commit()
finally:
await database.close()
return len(rows)
def main() -> None:
parser = argparse.ArgumentParser(description="Idempotently seed application settings")
parser.add_argument("--file", required=True, type=Path)
args = parser.parse_args()
count = asyncio.run(seed(args.file))
print(f"Application settings seeded: {count}")
if __name__ == "__main__":
main()
@@ -0,0 +1,40 @@
from __future__ import annotations
import asyncio
from sqlalchemy import select
from app.db import AppSetting, Database
from app.services import REQUIRED_SETTINGS
from app.settings import get_settings
async def validate() -> int:
database = Database(get_settings().database_url)
try:
async with database.sessions() as session:
active_keys = set(
(
await session.execute(
select(AppSetting.setting_key).where(AppSetting.record_status == "A")
)
)
.scalars()
.all()
)
finally:
await database.close()
missing = sorted(REQUIRED_SETTINGS - active_keys)
if missing:
raise RuntimeError(f"Mandatory application settings are missing: {', '.join(missing)}")
return len(REQUIRED_SETTINGS)
def main() -> None:
count = asyncio.run(validate())
print(f"Mandatory application settings validated: {count}")
if __name__ == "__main__":
main()
+358
View File
@@ -0,0 +1,358 @@
import uuid
from collections.abc import AsyncIterator
from datetime import datetime
from typing import Any
from sqlalchemy import (
JSON,
BigInteger,
Boolean,
CheckConstraint,
DateTime,
ForeignKey,
Index,
Integer,
String,
Text,
UniqueConstraint,
func,
)
from sqlalchemy.dialects.postgresql import INET, UUID
from sqlalchemy.ext.asyncio import (
AsyncEngine,
AsyncSession,
async_sessionmaker,
create_async_engine,
)
from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column
SCHEMA = "han_app"
class Base(DeclarativeBase):
type_annotation_map = {dict[str, Any]: JSON}
class Common:
id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
record_status: Mapped[str] = mapped_column(String(1), default="A", server_default="A")
status_changed_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
status_change_reason: Mapped[str | None] = mapped_column(String(255))
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
updated_at: Mapped[datetime] = mapped_column(
DateTime(timezone=True), server_default=func.now(), onupdate=func.now()
)
updater_user_id: Mapped[uuid.UUID | None] = mapped_column(UUID(as_uuid=True))
class UserIdentity(Common, Base):
__tablename__ = "user_identities"
__table_args__ = (Index("ix_user_identities_phone", "phone_number"), {"schema": SCHEMA})
keycloak_sub: Mapped[str] = mapped_column(String(255), unique=True)
phone_number: Mapped[str] = mapped_column(String(32))
last_login_at: Mapped[datetime] = mapped_column(DateTime(timezone=True))
class ClientProfile(Common, Base):
__tablename__ = "client_profiles"
__table_args__ = ({"schema": SCHEMA},)
user_id: Mapped[uuid.UUID] = mapped_column(
ForeignKey(f"{SCHEMA}.user_identities.id", ondelete="RESTRICT"), unique=True
)
bitrix_contact_id: Mapped[str | None] = mapped_column(String(64))
full_name: Mapped[str | None] = mapped_column(String(255))
citizenship: Mapped[str | None] = mapped_column(String(128))
russian_phone: Mapped[str | None] = mapped_column(String(32))
foreign_phone: Mapped[str | None] = mapped_column(String(32))
email: Mapped[str | None] = mapped_column(String(320))
source_updated_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
class UserConsent(Common, Base):
__tablename__ = "user_consents"
__table_args__ = (
UniqueConstraint("user_id", "consent_type", "document_version"),
CheckConstraint("consent_type IN ('personal_data','user_agreement','marketing')"),
{"schema": SCHEMA},
)
user_id: Mapped[uuid.UUID] = mapped_column(
ForeignKey(f"{SCHEMA}.user_identities.id", ondelete="RESTRICT")
)
ux_session_id: Mapped[uuid.UUID | None] = mapped_column(UUID(as_uuid=True))
consent_type: Mapped[str] = mapped_column(String(32))
document_version: Mapped[str] = mapped_column(String(64))
accepted: Mapped[bool] = mapped_column(Boolean)
accepted_at: Mapped[datetime] = mapped_column(DateTime(timezone=True))
client_ip: Mapped[str | None] = mapped_column(INET)
user_agent_hash: Mapped[str | None] = mapped_column(String(64))
class UxSession(Common, Base):
__tablename__ = "ux_sessions"
__table_args__ = (
CheckConstraint("start_reason IN ('first_launch','cold_start','idle_timeout')"),
Index("ix_ux_sessions_user_started", "user_id", "started_at"),
{"schema": SCHEMA},
)
user_id: Mapped[uuid.UUID] = mapped_column(
ForeignKey(f"{SCHEMA}.user_identities.id", ondelete="RESTRICT")
)
start_reason: Mapped[str] = mapped_column(String(32))
platform: Mapped[str] = mapped_column(String(32))
app_version: Mapped[str] = mapped_column(String(64))
device_id_hash: Mapped[str | None] = mapped_column(String(64))
started_at: Mapped[datetime] = mapped_column(DateTime(timezone=True))
class Dialog(Common, Base):
__tablename__ = "dialogs"
__table_args__ = (
CheckConstraint("status IN ('open','waiting_for_company','waiting_for_client','closed')"),
Index("ix_dialogs_user_updated", "user_id", "updated_at", "id"),
{"schema": SCHEMA},
)
user_id: Mapped[uuid.UUID] = mapped_column(
ForeignKey(f"{SCHEMA}.user_identities.id", ondelete="RESTRICT")
)
status: Mapped[str] = mapped_column(String(32), default="open")
last_message_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
closed_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
class Message(Common, Base):
__tablename__ = "messages"
__table_args__ = (
CheckConstraint("sender_type IN ('client','company')"),
CheckConstraint("content_kind IN ('text','file')"),
CheckConstraint("safety_status IN ('pending','allowed','blocked','needs_review')"),
CheckConstraint(
"delivery_status IN ('accepted','processing','delivered','rejected','failed')"
),
Index("ix_messages_dialog_created", "dialog_id", "created_at", "id"),
UniqueConstraint("dialog_id", "client_idempotency_key"),
{"schema": SCHEMA},
)
dialog_id: Mapped[uuid.UUID] = mapped_column(
ForeignKey(f"{SCHEMA}.dialogs.id", ondelete="RESTRICT")
)
sender_type: Mapped[str] = mapped_column(String(16))
content_kind: Mapped[str] = mapped_column(String(16))
text: Mapped[str] = mapped_column(Text, default="")
safety_status: Mapped[str] = mapped_column(String(16))
delivery_status: Mapped[str] = mapped_column(String(16))
external_message_id: Mapped[str | None] = mapped_column(String(255))
client_idempotency_key: Mapped[str | None] = mapped_column(String(128))
occurred_at: Mapped[datetime] = mapped_column(DateTime(timezone=True))
class MessageAttachment(Common, Base):
__tablename__ = "message_attachments"
__table_args__ = (
CheckConstraint("direction IN ('client_upload','company_inbound')"),
CheckConstraint("scan_status IN ('pending','clean','infected','failed')"),
CheckConstraint("size_bytes > 0"),
{"schema": SCHEMA},
)
dialog_id: Mapped[uuid.UUID] = mapped_column(ForeignKey(f"{SCHEMA}.dialogs.id"))
message_id: Mapped[uuid.UUID | None] = mapped_column(
ForeignKey(f"{SCHEMA}.messages.id"), unique=True
)
owner_user_id: Mapped[uuid.UUID] = mapped_column(ForeignKey(f"{SCHEMA}.user_identities.id"))
direction: Mapped[str] = mapped_column(String(32))
original_file_name: Mapped[str] = mapped_column(String(255))
safe_file_name: Mapped[str] = mapped_column(String(255))
mime_type: Mapped[str] = mapped_column(String(128))
size_bytes: Mapped[int] = mapped_column(BigInteger)
checksum_sha256: Mapped[str | None] = mapped_column(String(64))
scan_status: Mapped[str] = mapped_column(String(16), default="pending")
storage_bucket: Mapped[str] = mapped_column(String(255))
object_key: Mapped[str] = mapped_column(String(1024))
quarantine_object_key: Mapped[str | None] = mapped_column(String(1024))
upload_expires_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
completed_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
class Document(Common, Base):
__tablename__ = "documents"
__table_args__ = (UniqueConstraint("storage_bucket", "object_key"), {"schema": SCHEMA})
user_id: Mapped[uuid.UUID] = mapped_column(ForeignKey(f"{SCHEMA}.user_identities.id"))
name: Mapped[str] = mapped_column(String(255))
mime_type: Mapped[str] = mapped_column(String(128))
size_bytes: Mapped[int] = mapped_column(BigInteger)
checksum_sha256: Mapped[str] = mapped_column(String(64))
storage_bucket: Mapped[str] = mapped_column(String(255))
object_key: Mapped[str] = mapped_column(String(1024))
sent_at: Mapped[datetime] = mapped_column(DateTime(timezone=True))
class SafetyTask(Base):
__tablename__ = "safety_tasks"
__table_args__ = ({"schema": SCHEMA},)
id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
task_id: Mapped[str] = mapped_column(String(255), unique=True)
message_id: Mapped[uuid.UUID] = mapped_column(ForeignKey(f"{SCHEMA}.messages.id"), unique=True)
attachment_id: Mapped[uuid.UUID | None] = mapped_column(UUID(as_uuid=True))
quarantine_object_key: Mapped[str | None] = mapped_column(String(1024))
status: Mapped[str] = mapped_column(String(16))
deadline_at: Mapped[datetime] = mapped_column(DateTime(timezone=True))
next_poll_at: Mapped[datetime] = mapped_column(DateTime(timezone=True))
attempt_count: Mapped[int] = mapped_column(Integer, default=0)
last_error_code: Mapped[str | None] = mapped_column(String(64))
locked_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
locked_by: Mapped[str | None] = mapped_column(String(128))
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
class DeliveryOutbox(Base):
__tablename__ = "delivery_outbox"
__table_args__ = ({"schema": SCHEMA},)
id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
message_id: Mapped[uuid.UUID] = mapped_column(ForeignKey(f"{SCHEMA}.messages.id"), unique=True)
external_chat_id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True))
payload_json: Mapped[dict[str, Any]] = mapped_column(JSON)
status: Mapped[str] = mapped_column(String(16), default="pending")
attempt_count: Mapped[int] = mapped_column(Integer, default=0)
next_attempt_at: Mapped[datetime] = mapped_column(DateTime(timezone=True))
last_error_code: Mapped[str | None] = mapped_column(String(64))
locked_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
locked_by: Mapped[str | None] = mapped_column(String(128))
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
class IdempotencyRecord(Base):
__tablename__ = "idempotency_records"
__table_args__ = (
UniqueConstraint("scope", "user_id", "idempotency_key"),
{"schema": SCHEMA},
)
id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
scope: Mapped[str] = mapped_column(String(128))
user_id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True))
idempotency_key: Mapped[str] = mapped_column(String(128))
request_fingerprint: Mapped[str] = mapped_column(String(64))
status: Mapped[str] = mapped_column(String(16))
response_status: Mapped[int | None] = mapped_column(Integer)
response_body_json: Mapped[dict[str, Any] | None] = mapped_column(JSON)
resource_type: Mapped[str | None] = mapped_column(String(64))
resource_id: Mapped[uuid.UUID | None] = mapped_column(UUID(as_uuid=True))
expires_at: Mapped[datetime] = mapped_column(DateTime(timezone=True))
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
class OpenLinesInboxReceipt(Base):
__tablename__ = "openlines_inbox_receipts"
__table_args__ = (
UniqueConstraint("event_id"),
UniqueConstraint("external_chat_id", "bitrix_message_id"),
{"schema": SCHEMA},
)
id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
event_id: Mapped[str] = mapped_column(String(255))
external_chat_id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True))
bitrix_message_id: Mapped[str | None] = mapped_column(String(255))
event_type: Mapped[str] = mapped_column(String(32))
payload_fingerprint: Mapped[str] = mapped_column(String(64))
status: Mapped[str] = mapped_column(String(16))
message_id: Mapped[uuid.UUID | None] = mapped_column(UUID(as_uuid=True))
last_error_code: Mapped[str | None] = mapped_column(String(64))
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
class AuditEvent(Base):
__tablename__ = "audit_events"
__table_args__ = ({"schema": SCHEMA},)
id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
event_type: Mapped[str] = mapped_column(String(128))
actor_type: Mapped[str] = mapped_column(String(32))
user_id: Mapped[uuid.UUID | None] = mapped_column(UUID(as_uuid=True))
ux_session_id: Mapped[uuid.UUID | None] = mapped_column(UUID(as_uuid=True))
request_id: Mapped[str] = mapped_column(String(64))
trace_id: Mapped[str | None] = mapped_column(String(64))
resource_type: Mapped[str | None] = mapped_column(String(64))
resource_id: Mapped[uuid.UUID | None] = mapped_column(UUID(as_uuid=True))
ip: Mapped[str | None] = mapped_column(INET)
user_agent_hash: Mapped[str | None] = mapped_column(String(64))
outcome: Mapped[str] = mapped_column(String(32))
metadata_json: Mapped[dict[str, Any]] = mapped_column(JSON, default=dict)
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
class AppSetting(Base):
__tablename__ = "app_settings"
__table_args__ = ({"schema": SCHEMA},)
setting_key: Mapped[str] = mapped_column(String(255), primary_key=True)
setting_value: Mapped[str] = mapped_column(Text)
value_type: Mapped[str] = mapped_column(String(32))
is_public: Mapped[bool] = mapped_column(Boolean, default=False)
description: Mapped[str | None] = mapped_column(Text)
record_status: Mapped[str] = mapped_column(String(1), default="A")
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
class TextResource(Common, Base):
__tablename__ = "text_resources"
__table_args__ = (
UniqueConstraint("mnemonic", "locale"),
{"schema": SCHEMA},
)
mnemonic: Mapped[str] = mapped_column(String(255))
locale: Mapped[str] = mapped_column(String(16), default="ru")
text_value: Mapped[str] = mapped_column(Text)
sort_order: Mapped[int] = mapped_column(Integer, default=0)
class PopularQuestion(Common, Base):
__tablename__ = "popular_questions"
__table_args__ = (
UniqueConstraint("mnemonic", "locale"),
{"schema": SCHEMA},
)
mnemonic: Mapped[str] = mapped_column(String(255))
locale: Mapped[str] = mapped_column(String(16), default="ru")
question_text: Mapped[str] = mapped_column(Text)
sort_order: Mapped[int] = mapped_column(Integer, default=0)
class SyncQueue(Base):
__tablename__ = "sync_queue"
__table_args__ = ({"schema": SCHEMA},)
id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
task_type: Mapped[str] = mapped_column(String(64))
entity_type: Mapped[str] = mapped_column(String(64))
entity_id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True))
dedup_key: Mapped[str] = mapped_column(String(255), unique=True)
payload_json: Mapped[dict[str, Any]] = mapped_column(JSON)
status: Mapped[str] = mapped_column(String(16), default="pending")
attempt_count: Mapped[int] = mapped_column(Integer, default=0)
next_attempt_at: Mapped[datetime] = mapped_column(
DateTime(timezone=True), server_default=func.now()
)
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
class EntityExternalMapping(Base):
__tablename__ = "entity_external_mapping"
__table_args__ = (UniqueConstraint("entity_type", "entity_id"), {"schema": SCHEMA})
id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
entity_type: Mapped[str] = mapped_column(String(64))
entity_id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True))
external_id: Mapped[str] = mapped_column(String(128))
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
class Database:
def __init__(self, url: str) -> None:
self.engine: AsyncEngine = create_async_engine(url, pool_pre_ping=True)
self.sessions = async_sessionmaker(self.engine, expire_on_commit=False)
async def session(self) -> AsyncIterator[AsyncSession]:
async with self.sessions() as session:
yield session
async def close(self) -> None:
await self.engine.dispose()
@@ -0,0 +1,338 @@
import asyncio
import hashlib
import ipaddress
import json
import socket
import time
import uuid
from dataclasses import dataclass
from typing import Any
from urllib.parse import urlparse
import boto3
import httpx
from botocore.config import Config
from redis.asyncio import Redis
from app.settings import Settings
RATE_LIMIT_LUA = """
local current = redis.call('INCR', KEYS[1])
if current == 1 then redis.call('EXPIRE', KEYS[1], ARGV[1]) end
local ttl = redis.call('TTL', KEYS[1])
return {current, ttl}
"""
class DependencyFailure(Exception):
def __init__(self, code: str = "dependency_unavailable", timeout: bool = False) -> None:
self.code = code
self.timeout = timeout
@dataclass(slots=True)
class CircuitBreaker:
threshold: int
open_seconds: float
failures: int = 0
opened_at: float | None = None
def allow(self) -> bool:
if self.opened_at is None:
return True
if time.monotonic() - self.opened_at >= self.open_seconds:
self.opened_at = None
self.failures = max(0, self.threshold - 1)
return True
return False
def success(self) -> None:
self.failures = 0
self.opened_at = None
def failure(self) -> None:
self.failures += 1
if self.failures >= self.threshold:
self.opened_at = time.monotonic()
class RateLimiter:
def __init__(self, redis: Redis) -> None:
self.redis = redis
async def consume(self, key: str, limit: int, window: int) -> int:
try:
count, ttl = await self.redis.eval(RATE_LIMIT_LUA, 1, key, window)
except Exception as exc:
raise DependencyFailure() from exc
if int(count) > limit:
return max(1, int(ttl))
return 0
@staticmethod
def key(identity_type: str, identity: str, route: str, window: int) -> str:
safe_identity = hashlib.sha256(identity.encode()).hexdigest()[:32]
bucket = int(time.time()) // window
return f"han:api:rl:{identity_type}:{safe_identity}:{route}:{bucket}"
class RedisIdempotency:
def __init__(self, redis: Redis) -> None:
self.redis = redis
async def get(self, scope: str, user_id: uuid.UUID, key: str) -> dict[str, Any] | None:
key_hash = hashlib.sha256(key.encode()).hexdigest()
raw = await self.redis.get(f"han:api:idem:{scope}:{user_id}:{key_hash}")
return json.loads(raw) if raw else None
async def put(self, scope: str, user_id: uuid.UUID, key: str, value: dict[str, Any]) -> None:
key_hash = hashlib.sha256(key.encode()).hexdigest()
await self.redis.set(
f"han:api:idem:{scope}:{user_id}:{key_hash}",
json.dumps(value, separators=(",", ":"), default=str),
ex=86400,
)
class SafetyClient:
def __init__(self, settings: Settings, http: httpx.AsyncClient) -> None:
self.settings = settings
self.http = http
self.breaker = CircuitBreaker(
settings.message_safety_circuit_failure_threshold,
settings.message_safety_circuit_open_sec,
)
async def check(self, payload: dict[str, Any], request_id: str) -> dict[str, Any]:
return await self._call(
"POST",
"/internal/safety/v1/messages/check",
request_id,
json=payload,
timeout=self.settings.message_safety_post_timeout_sec,
)
async def poll(self, task_id: str, request_id: str) -> dict[str, Any]:
return await self._call(
"GET",
f"/internal/safety/v1/messages/tasks/{task_id}",
request_id,
timeout=2,
)
async def _call(self, method: str, path: str, request_id: str, **kwargs: Any) -> dict[str, Any]:
if not self.breaker.allow():
raise DependencyFailure()
headers = {
"X-Service-Token": self.settings.message_safety_service_token.get_secret_value(),
"X-Request-ID": request_id,
}
try:
response = await self.http.request(
method,
f"{str(self.settings.message_safety_url).rstrip('/')}{path}",
headers=headers,
**kwargs,
)
except httpx.TimeoutException as exc:
self.breaker.failure()
raise DependencyFailure(timeout=True) from exc
except httpx.HTTPError as exc:
self.breaker.failure()
raise DependencyFailure() from exc
if response.status_code == 401 or response.status_code >= 500:
self.breaker.failure()
raise DependencyFailure()
try:
body = response.json()
except ValueError as exc:
self.breaker.failure()
raise DependencyFailure() from exc
if not isinstance(body, dict):
self.breaker.failure()
raise DependencyFailure()
self.breaker.success()
body["_status"] = response.status_code
return body
class OpenLinesClient:
def __init__(self, settings: Settings, http: httpx.AsyncClient) -> None:
self.settings = settings
self.http = http
self.breaker = CircuitBreaker(
settings.bitrix_local_app_circuit_failure_threshold,
settings.bitrix_local_app_circuit_open_sec,
)
async def send(
self, message_id: uuid.UUID, payload: dict[str, Any], request_id: str
) -> dict[str, Any]:
if not self.breaker.allow():
raise DependencyFailure()
try:
response = await self.http.post(
f"{str(self.settings.bitrix_local_app_base_url).rstrip('/')}"
"/internal/openlines/v1/messages",
json=payload,
headers={
"Authorization": "Bearer "
+ self.settings.bitrix_local_app_internal_token.get_secret_value(),
"Idempotency-Key": str(message_id),
"X-Request-ID": request_id,
},
timeout=self.settings.bitrix_local_app_http_timeout_sec,
)
response.raise_for_status()
except httpx.TimeoutException as exc:
self.breaker.failure()
raise DependencyFailure(timeout=True) from exc
except httpx.HTTPError as exc:
self.breaker.failure()
raise DependencyFailure() from exc
self.breaker.success()
return response.json()
async def ready(self) -> bool:
try:
response = await self.http.get(
f"{str(self.settings.bitrix_local_app_base_url).rstrip('/')}"
"/internal/openlines/v1/status",
headers={
"Authorization": "Bearer "
+ self.settings.bitrix_local_app_internal_token.get_secret_value()
},
timeout=2,
)
return response.is_success
except httpx.HTTPError:
return False
async def fresh_openlines_payload(
payload: dict[str, Any], s3: "S3Client"
) -> dict[str, Any]:
result = json.loads(json.dumps(payload, default=str))
for file in result.get("message", {}).get("files", []):
bucket = file.pop("_storage_bucket")
key = file.pop("_object_key")
file["download_url"] = await s3.presign_get(bucket, key)
return result
class S3Client:
def __init__(self, settings: Settings) -> None:
self.settings = settings
self.client = boto3.client(
"s3",
endpoint_url=str(settings.selectel_s3_endpoint_url),
aws_access_key_id=settings.selectel_s3_access_key.get_secret_value(),
aws_secret_access_key=settings.selectel_s3_secret_key.get_secret_value(),
config=Config(connect_timeout=3, read_timeout=10, retries={"max_attempts": 2}),
)
async def ready(self) -> bool:
try:
for bucket in (
self.settings.selectel_s3_bucket_quarantine,
self.settings.selectel_s3_bucket_attachments,
self.settings.selectel_s3_bucket_documents,
):
await asyncio.to_thread(self.client.head_bucket, Bucket=bucket)
return True
except Exception:
return False
async def presign_put(self, key: str, mime: str, ttl: int) -> str:
return await asyncio.to_thread(
self.client.generate_presigned_url,
"put_object",
Params={
"Bucket": self.settings.selectel_s3_bucket_quarantine,
"Key": key,
"ContentType": mime,
},
ExpiresIn=ttl,
)
async def presign_get(self, bucket: str, key: str, ttl: int = 300) -> str:
return await asyncio.to_thread(
self.client.generate_presigned_url,
"get_object",
Params={"Bucket": bucket, "Key": key},
ExpiresIn=ttl,
)
async def head(self, bucket: str, key: str) -> dict[str, Any]:
return await asyncio.to_thread(self.client.head_object, Bucket=bucket, Key=key)
async def promote(self, source_key: str, destination_key: str) -> None:
await asyncio.to_thread(
self.client.copy_object,
Bucket=self.settings.selectel_s3_bucket_attachments,
Key=destination_key,
CopySource={
"Bucket": self.settings.selectel_s3_bucket_quarantine,
"Key": source_key,
},
)
await asyncio.to_thread(
self.client.delete_object,
Bucket=self.settings.selectel_s3_bucket_quarantine,
Key=source_key,
)
async def delete_quarantine(self, key: str) -> None:
await asyncio.to_thread(
self.client.delete_object,
Bucket=self.settings.selectel_s3_bucket_quarantine,
Key=key,
)
async def upload_inbound(
self,
http: httpx.AsyncClient,
download_url: str,
destination_key: str,
mime_type: str,
max_bytes: int,
) -> tuple[int, str]:
parsed = urlparse(download_url)
if parsed.scheme != "https" or not parsed.hostname:
raise DependencyFailure("unsafe_inbound_url")
try:
addresses = await asyncio.to_thread(
socket.getaddrinfo, parsed.hostname, parsed.port or 443, type=socket.SOCK_STREAM
)
except OSError as exc:
raise DependencyFailure("unsafe_inbound_url") from exc
if any(
ipaddress.ip_address(address[4][0]).is_private
or ipaddress.ip_address(address[4][0]).is_loopback
or ipaddress.ip_address(address[4][0]).is_link_local
or ipaddress.ip_address(address[4][0]).is_reserved
for address in addresses
):
raise DependencyFailure("unsafe_inbound_url")
data = bytearray()
try:
async with http.stream(
"GET", download_url, timeout=10, follow_redirects=False
) as response:
response.raise_for_status()
if response.headers.get("content-type", "").split(";")[0] != mime_type:
raise DependencyFailure("inbound_mime_mismatch")
async for chunk in response.aiter_bytes():
data.extend(chunk)
if len(data) > max_bytes:
raise DependencyFailure("inbound_file_too_large")
except httpx.HTTPError as exc:
raise DependencyFailure() from exc
await asyncio.to_thread(
self.client.put_object,
Bucket=self.settings.selectel_s3_bucket_attachments,
Key=destination_key,
Body=bytes(data),
ContentType=mime_type,
)
return len(data), hashlib.sha256(data).hexdigest()
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,65 @@
import asyncio
import json
import uuid
from collections.abc import AsyncIterator
from contextlib import suppress
from typing import Any
from redis.asyncio import Redis
CHANNEL_PREFIX = "han:realtime:dialog:"
class LocalFanout:
def __init__(self) -> None:
self._queues: set[asyncio.Queue[dict[str, Any]]] = set()
async def publish(self, event: dict[str, Any]) -> None:
for queue in tuple(self._queues):
with suppress(asyncio.QueueFull):
queue.put_nowait(event)
async def subscribe(self) -> AsyncIterator[dict[str, Any]]:
queue: asyncio.Queue[dict[str, Any]] = asyncio.Queue(maxsize=256)
self._queues.add(queue)
try:
while True:
yield await queue.get()
finally:
self._queues.discard(queue)
class RealtimeFanout:
def __init__(self, redis: Redis, local: LocalFanout | None = None) -> None:
self.redis = redis
self.local = local or LocalFanout()
async def publish(self, event: dict[str, Any]) -> None:
event = {"event_id": str(uuid.uuid4()), **event}
channel = CHANNEL_PREFIX + str(event["dialog_id"])
try:
await self.redis.publish(channel, json.dumps(event, default=str, separators=(",", ":")))
except Exception:
await self.local.publish(event)
async def events(self, dialog_ids: set[uuid.UUID]) -> AsyncIterator[dict[str, Any]]:
channels = [CHANNEL_PREFIX + str(dialog_id) for dialog_id in dialog_ids]
pubsub = self.redis.pubsub()
try:
await pubsub.subscribe(*channels)
except Exception:
await pubsub.aclose()
async for event in self.local.subscribe():
if uuid.UUID(str(event["dialog_id"])) in dialog_ids:
yield event
return
try:
while True:
message = await pubsub.get_message(ignore_subscribe_messages=True, timeout=1)
if message:
yield json.loads(message["data"])
else:
await asyncio.sleep(0)
finally:
await pubsub.unsubscribe(*channels)
await pubsub.aclose()
+148
View File
@@ -0,0 +1,148 @@
import hashlib
import hmac
import json
import uuid
from base64 import urlsafe_b64decode, urlsafe_b64encode
from datetime import datetime
from enum import StrEnum
from typing import Annotated, Literal
from pydantic import BaseModel, ConfigDict, Field, HttpUrl, model_validator
class StrictModel(BaseModel):
model_config = ConfigDict(extra="forbid")
class Device(StrictModel):
platform: Literal["ios", "android", "web"]
app_version: str = Field(min_length=1, max_length=64)
device_id: str | None = Field(default=None, max_length=255)
class ConsentChoice(StrictModel):
accepted: bool
version: str = Field(min_length=1, max_length=64)
class ConsentSet(StrictModel):
personal_data: ConsentChoice
user_agreement: ConsentChoice
marketing: ConsentChoice
class BootstrapRequest(StrictModel):
consents: ConsentSet
device: Device
class ConsentsRequest(StrictModel):
consents: ConsentSet
class SessionStartRequest(StrictModel):
start_reason: Literal["first_launch", "cold_start", "idle_timeout"]
device: Device
class TextMessageRequest(StrictModel):
content_kind: Literal["text"]
text: str = Field(min_length=1, max_length=4000)
class FileMessageRequest(StrictModel):
content_kind: Literal["file"]
attachment_id: uuid.UUID
checksum: str = Field(pattern=r"^sha256:[0-9a-f]{64}$")
MessageRequest = Annotated[
TextMessageRequest | FileMessageRequest, Field(discriminator="content_kind")
]
class AttachmentInitRequest(StrictModel):
file_name: str = Field(min_length=1, max_length=255)
mime_type: str = Field(min_length=1, max_length=128)
size_bytes: int = Field(gt=0)
class AttachmentCompleteRequest(StrictModel):
checksum: str = Field(pattern=r"^sha256:[0-9a-f]{64}$")
class OpenLinesFile(StrictModel):
name: str = Field(min_length=1, max_length=255)
mime_type: str = Field(min_length=1, max_length=128)
size_bytes: int = Field(gt=0)
download_url: HttpUrl
class OpenLinesMessage(StrictModel):
text: str = Field(default="", max_length=4000)
files: list[OpenLinesFile] = Field(default_factory=list, max_length=1)
@model_validator(mode="after")
def non_empty(self) -> "OpenLinesMessage":
if not self.text.strip() and not self.files:
raise ValueError("message must contain text or file")
return self
class OpenLinesInbox(StrictModel):
event_id: str = Field(min_length=1, max_length=255)
event_type: Literal["message.new", "dialog.closed"]
external_chat_id: uuid.UUID
bitrix_message_id: str | None = Field(default=None, max_length=255)
occurred_at: datetime
message: OpenLinesMessage | None = None
@model_validator(mode="after")
def event_shape(self) -> "OpenLinesInbox":
if self.event_type == "message.new" and (
not self.bitrix_message_id or self.message is None
):
raise ValueError("message.new requires bitrix_message_id and message")
return self
class DialogStatus(StrEnum):
OPEN = "open"
WAITING_COMPANY = "waiting_for_company"
WAITING_CLIENT = "waiting_for_client"
CLOSED = "closed"
def canonical_fingerprint(
method: str, route: str, path_params: dict[str, str], body: object, user_id: uuid.UUID
) -> str:
value = {
"method": method.upper(),
"route": route,
"path": dict(sorted(path_params.items())),
"body": body,
"user_id": str(user_id),
}
return hashlib.sha256(
json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=False).encode()
).hexdigest()
def encode_cursor(data: dict[str, str], secret: bytes) -> str:
payload = json.dumps({"v": 1, **data}, sort_keys=True, separators=(",", ":")).encode()
signature = hmac.digest(secret, payload, "sha256")
return urlsafe_b64encode(payload + signature).decode().rstrip("=")
def decode_cursor(value: str, secret: bytes) -> dict[str, str]:
try:
raw = urlsafe_b64decode(value + "=" * (-len(value) % 4))
payload, signature = raw[:-32], raw[-32:]
if not hmac.compare_digest(signature, hmac.digest(secret, payload, "sha256")):
raise ValueError("invalid cursor")
decoded = json.loads(payload)
if decoded.pop("v") != 1:
raise ValueError("unsupported cursor")
return decoded
except (ValueError, KeyError, json.JSONDecodeError) as exc:
raise ValueError("invalid cursor") from exc
@@ -0,0 +1,930 @@
import hashlib
import json
import re
import unicodedata
import uuid
from dataclasses import dataclass
from datetime import UTC, datetime, timedelta
from pathlib import PurePath
from typing import Any
import httpx
from sqlalchemy import func, select
from sqlalchemy.dialects.postgresql import insert
from sqlalchemy.ext.asyncio import AsyncSession
from app.auth import Principal
from app.db import (
AppSetting,
AuditEvent,
ClientProfile,
DeliveryOutbox,
Dialog,
Document,
IdempotencyRecord,
Message,
MessageAttachment,
OpenLinesInboxReceipt,
SafetyTask,
UserConsent,
UserIdentity,
UxSession,
)
from app.integrations import (
DependencyFailure,
OpenLinesClient,
S3Client,
SafetyClient,
fresh_openlines_payload,
)
from app.realtime import RealtimeFanout
from app.schemas import (
AttachmentCompleteRequest,
AttachmentInitRequest,
BootstrapRequest,
ConsentsRequest,
FileMessageRequest,
MessageRequest,
OpenLinesInbox,
SessionStartRequest,
encode_cursor,
)
from app.settings import Settings
class DomainError(Exception):
def __init__(self, code: str, status: int, message: str, details: dict[str, Any] | None = None):
self.code, self.status, self.message = code, status, message
self.details = details or {}
REQUIRED_SETTINGS = {
"auth.phone.enabled",
"auth.password.enabled",
"otp.phone.max_send_attempts_per_24h",
"otp.phone.min_seconds_between_attempts",
"operator.call.phone",
"consent.personal_data.required",
"consent.personal_data.document_url",
"consent.personal_data.version",
"consent.user_agreement.required",
"consent.user_agreement.document_url",
"consent.user_agreement.version",
"consent.marketing.required",
"consent.marketing.version",
"chat.attachments.allowed_extensions",
"chat.attachments.allowed_mime_types",
"chat.attachments.max_size_mb",
"chat.attachments.presigned_upload_ttl_seconds",
"rate_limit.message_send.per_user",
"rate_limit.message_send.per_dialog",
"rate_limit.download_url.per_user",
"rate_limit.public_endpoints.per_ip",
"ux.session.idle_timeout_minutes",
"security.cors.allowed_origins",
"security.public_cache.max_age_seconds",
}
@dataclass(frozen=True, slots=True)
class SettingsSnapshot:
values: dict[str, str]
version: str
def boolean(self, key: str) -> bool:
return self.values[key].lower() == "true"
def integer(self, key: str) -> int:
return int(self.values[key])
def strings(self, key: str) -> list[str]:
return [item.strip() for item in self.values[key].split(",") if item.strip()]
def limit(self, key: str) -> tuple[int, int]:
amount, period = self.values[key].split("/", 1)
windows = {"second": 1, "minute": 60, "hour": 3600, "day": 86400}
return int(amount), windows[period]
async def load_settings(session: AsyncSession) -> SettingsSnapshot:
rows = (
await session.execute(select(AppSetting).where(AppSetting.record_status == "A"))
).scalars()
values = {row.setting_key: row.setting_value for row in rows}
missing = REQUIRED_SETTINGS - values.keys()
if missing:
raise DomainError(
"dependency_unavailable",
503,
"Required settings are unavailable",
{"missing": sorted(missing)},
)
version = hashlib.sha256(json.dumps(values, sort_keys=True).encode()).hexdigest()[:24]
return SettingsSnapshot(values, version)
async def resolve_user(session: AsyncSession, principal: Principal) -> UserIdentity:
user = (
await session.execute(
select(UserIdentity).where(
UserIdentity.keycloak_sub == principal.subject, UserIdentity.record_status == "A"
)
)
).scalar_one_or_none()
if user is None:
raise DomainError("resource_state_conflict", 409, "Bootstrap required")
return user
def audit(
event_type: str,
request_id: str,
user_id: uuid.UUID | None,
resource_type: str | None = None,
resource_id: uuid.UUID | None = None,
ux_session_id: uuid.UUID | None = None,
metadata: dict[str, Any] | None = None,
) -> AuditEvent:
return AuditEvent(
event_type=event_type,
actor_type="user" if user_id else "service",
user_id=user_id,
ux_session_id=ux_session_id,
request_id=request_id,
resource_type=resource_type,
resource_id=resource_id,
outcome="success",
metadata_json=metadata or {},
)
def validate_consents(consents: Any, snapshot: SettingsSnapshot) -> None:
for name in ("personal_data", "user_agreement", "marketing"):
choice = getattr(consents, name)
if choice.version != snapshot.values[f"consent.{name}.version"]:
raise DomainError(
"validation_error", 400, "Consent version is not current", {"field": name}
)
if snapshot.boolean(f"consent.{name}.required") and not choice.accepted:
raise DomainError("consents_required", 403, "Required consents must be accepted")
async def bootstrap(
session: AsyncSession,
principal: Principal,
body: BootstrapRequest,
snapshot: SettingsSnapshot,
request_id: str,
) -> dict[str, Any]:
if not principal.phone_number:
raise DomainError("phone_claim_missing", 400, "Verified phone claim is missing")
validate_consents(body.consents, snapshot)
now = datetime.now(UTC)
statement = (
insert(UserIdentity)
.values(
id=uuid.uuid4(),
keycloak_sub=principal.subject,
phone_number=principal.phone_number,
last_login_at=now,
)
.on_conflict_do_update(
index_elements=[UserIdentity.keycloak_sub],
set_={"phone_number": principal.phone_number, "last_login_at": now, "updated_at": now},
)
.returning(UserIdentity.id)
)
user_id = (await session.execute(statement)).scalar_one()
await session.execute(
insert(ClientProfile)
.values(id=uuid.uuid4(), user_id=user_id)
.on_conflict_do_nothing(index_elements=[ClientProfile.user_id])
)
for consent_type in ("personal_data", "user_agreement", "marketing"):
choice = getattr(body.consents, consent_type)
await session.execute(
insert(UserConsent)
.values(
id=uuid.uuid4(),
user_id=user_id,
consent_type=consent_type,
document_version=choice.version,
accepted=choice.accepted,
accepted_at=now,
)
.on_conflict_do_nothing(
index_elements=[
UserConsent.user_id,
UserConsent.consent_type,
UserConsent.document_version,
]
)
)
session.add(audit("auth.bootstrap", request_id, user_id))
await session.commit()
return {"user_id": user_id, "profile_ready": True}
async def record_consents(
session: AsyncSession,
user: UserIdentity,
body: ConsentsRequest,
snapshot: SettingsSnapshot,
request_id: str,
ux_session_id: uuid.UUID | None,
) -> dict[str, Any]:
validate_consents(body.consents, snapshot)
now = datetime.now(UTC)
versions: dict[str, str] = {}
for consent_type in ("personal_data", "user_agreement", "marketing"):
choice = getattr(body.consents, consent_type)
versions[consent_type] = choice.version
await session.execute(
insert(UserConsent)
.values(
id=uuid.uuid4(),
user_id=user.id,
ux_session_id=ux_session_id,
consent_type=consent_type,
document_version=choice.version,
accepted=choice.accepted,
accepted_at=now,
)
.on_conflict_do_nothing()
)
session.add(audit("consent.recorded", request_id, user.id, ux_session_id=ux_session_id))
await session.commit()
return {"recorded_at": now, "versions": versions}
async def start_session(
session: AsyncSession, user: UserIdentity, body: SessionStartRequest, request_id: str
) -> dict[str, Any]:
now, session_id = datetime.now(UTC), uuid.uuid4()
device_hash = (
hashlib.sha256(body.device.device_id.encode()).hexdigest()
if body.device.device_id
else None
)
session.add(
UxSession(
id=session_id,
user_id=user.id,
start_reason=body.start_reason,
platform=body.device.platform,
app_version=body.device.app_version,
device_id_hash=device_hash,
started_at=now,
)
)
session.add(audit("session_start", request_id, user.id, ux_session_id=session_id))
await session.commit()
return {"ux_session_id": session_id, "started_at": now}
async def get_profile(session: AsyncSession, user: UserIdentity) -> dict[str, Any]:
profile = (
await session.execute(
select(ClientProfile).where(
ClientProfile.user_id == user.id, ClientProfile.record_status == "A"
)
)
).scalar_one()
document_count = (
await session.scalar(
select(func.count(Document.id)).where(
Document.user_id == user.id, Document.record_status == "A"
)
)
or 0
)
return {
"user_id": user.id,
"profile": {
"personal_data": {
"full_name": profile.full_name,
"citizenship": profile.citizenship,
"russian_phone": profile.russian_phone,
"foreign_phone": profile.foreign_phone,
"email": profile.email,
},
"documents": {"count": document_count},
},
}
def dialog_dto(dialog: Dialog) -> dict[str, Any]:
return {
"dialog_id": dialog.id,
"status": dialog.status,
"last_message_preview": None,
"unread_count": 0,
"created_at": dialog.created_at,
"updated_at": dialog.updated_at,
}
def message_dto(
message: Message, attachments: list[MessageAttachment] | None = None
) -> dict[str, Any]:
return {
"message_id": message.id,
"dialog_id": message.dialog_id,
"sender_type": message.sender_type,
"content_kind": message.content_kind,
"text": message.text,
"attachments": [
{
"attachment_id": item.id,
"file_name": item.safe_file_name,
"mime_type": item.mime_type,
"size_bytes": item.size_bytes,
"scan_status": item.scan_status,
}
for item in (attachments or [])
],
"safety_status": message.safety_status,
"delivery_status": message.delivery_status,
"created_at": message.created_at,
}
def message_cursor(message: Message, settings: Settings) -> str:
return encode_cursor(
{"created_at": message.created_at.isoformat(), "id": str(message.id)},
settings.cursor_hmac_secret.get_secret_value().encode(),
)
async def publish_message(
fanout: RealtimeFanout,
message: Message,
settings: Settings,
attachments: list[MessageAttachment] | None = None,
) -> None:
await fanout.publish(
{
"type": "message.new",
"dialog_id": str(message.dialog_id),
"message": message_dto(message, attachments),
"cursor": message_cursor(message, settings),
}
)
async def publish_message_status(
fanout: RealtimeFanout, message: Message, settings: Settings
) -> None:
await fanout.publish(
{
"type": "message.status",
"dialog_id": str(message.dialog_id),
"message_id": str(message.id),
"safety_status": message.safety_status,
"delivery_status": message.delivery_status,
"cursor": message_cursor(message, settings),
}
)
async def publish_dialog_status(fanout: RealtimeFanout, dialog: Dialog) -> None:
await fanout.publish(
{"type": "dialog.status", "dialog_id": str(dialog.id), "status": dialog.status}
)
async def owned_dialog(session: AsyncSession, user_id: uuid.UUID, dialog_id: uuid.UUID) -> Dialog:
dialog = (
await session.execute(
select(Dialog).where(
Dialog.id == dialog_id, Dialog.user_id == user_id, Dialog.record_status == "A"
)
)
).scalar_one_or_none()
if dialog is None:
raise DomainError("not_found", 404, "Resource was not found")
return dialog
async def create_dialog(
session: AsyncSession, user: UserIdentity, request_id: str, idempotency_key: str
) -> tuple[dict[str, Any], int]:
scope = "dialogs.create"
fingerprint = hashlib.sha256(f"POST:/api/v1/dialogs:{user.id}".encode()).hexdigest()
record = (
await session.execute(
select(IdempotencyRecord).where(
IdempotencyRecord.scope == scope,
IdempotencyRecord.user_id == user.id,
IdempotencyRecord.idempotency_key == idempotency_key,
)
)
).scalar_one_or_none()
if record:
if record.request_fingerprint != fingerprint:
raise DomainError("idempotency_key_reused", 409, "Idempotency key was reused")
if record.status == "completed" and record.response_body_json:
return record.response_body_json, record.response_status or 200
existing = (
await session.execute(
select(Dialog).where(
Dialog.user_id == user.id,
Dialog.record_status == "A",
Dialog.status.in_(["open", "waiting_for_company", "waiting_for_client"]),
)
)
).scalar_one_or_none()
if existing:
result = dialog_dto(existing)
session.add(
IdempotencyRecord(
scope=scope,
user_id=user.id,
idempotency_key=idempotency_key,
request_fingerprint=fingerprint,
status="completed",
response_status=200,
response_body_json=json.loads(json.dumps(result, default=str)),
resource_type="dialog",
resource_id=existing.id,
expires_at=datetime.now(UTC) + timedelta(hours=24),
)
)
await session.commit()
return result, 200
dialog = Dialog(id=uuid.uuid4(), user_id=user.id, status="open")
session.add(dialog)
session.add(audit("dialog.created", request_id, user.id, "dialog", dialog.id))
result = dialog_dto(dialog)
session.add(
IdempotencyRecord(
scope=scope,
user_id=user.id,
idempotency_key=idempotency_key,
request_fingerprint=fingerprint,
status="completed",
response_status=201,
response_body_json=json.loads(json.dumps(result, default=str)),
resource_type="dialog",
resource_id=dialog.id,
expires_at=datetime.now(UTC) + timedelta(hours=24),
)
)
await session.commit()
await session.refresh(dialog)
return dialog_dto(dialog), 201
async def init_attachment(
session: AsyncSession,
user: UserIdentity,
dialog_id: uuid.UUID,
body: AttachmentInitRequest,
snapshot: SettingsSnapshot,
s3: S3Client,
request_id: str,
) -> dict[str, Any]:
await owned_dialog(session, user.id, dialog_id)
extension = PurePath(body.file_name).suffix.lower().lstrip(".")
if (
extension not in snapshot.strings("chat.attachments.allowed_extensions")
or body.mime_type not in snapshot.strings("chat.attachments.allowed_mime_types")
or body.size_bytes > snapshot.integer("chat.attachments.max_size_mb") * 1024 * 1024
):
raise DomainError("validation_error", 400, "File type or size is not allowed")
attachment_id = uuid.uuid4()
key = f"quarantine/users/{user.id}/dialogs/{dialog_id}/{attachment_id}"
ttl = snapshot.integer("chat.attachments.presigned_upload_ttl_seconds")
expires = datetime.now(UTC) + timedelta(seconds=ttl)
safe_name = re.sub(r"[^A-Za-z0-9._-]", "_", unicodedata.normalize("NFKC", body.file_name))
item = MessageAttachment(
id=attachment_id,
dialog_id=dialog_id,
owner_user_id=user.id,
direction="client_upload",
original_file_name=body.file_name,
safe_file_name=safe_name,
mime_type=body.mime_type,
size_bytes=body.size_bytes,
scan_status="pending",
storage_bucket=s3.settings.selectel_s3_bucket_quarantine,
object_key=key,
quarantine_object_key=key,
upload_expires_at=expires,
)
session.add(item)
session.add(
audit("attachment.upload_initialized", request_id, user.id, "attachment", attachment_id)
)
await session.commit()
url = await s3.presign_put(key, body.mime_type, ttl)
return {
"attachment_id": attachment_id,
"upload_url": url,
"upload_headers": {"Content-Type": body.mime_type},
"expires_at": expires,
}
async def complete_attachment(
session: AsyncSession,
user: UserIdentity,
dialog_id: uuid.UUID,
attachment_id: uuid.UUID,
body: AttachmentCompleteRequest,
s3: S3Client,
request_id: str,
) -> dict[str, Any]:
item = (
await session.execute(
select(MessageAttachment).where(
MessageAttachment.id == attachment_id,
MessageAttachment.dialog_id == dialog_id,
MessageAttachment.owner_user_id == user.id,
MessageAttachment.record_status == "A",
)
)
).scalar_one_or_none()
if item is None:
raise DomainError("not_found", 404, "Resource was not found")
checksum = body.checksum.removeprefix("sha256:")
if item.completed_at:
if item.checksum_sha256 != checksum:
raise DomainError("resource_state_conflict", 409, "Attachment checksum changed")
return attachment_dto(item)
try:
head = await s3.head(item.storage_bucket, item.object_key)
except Exception as exc:
raise DomainError("dependency_unavailable", 503, "Object storage is unavailable") from exc
if int(head["ContentLength"]) != item.size_bytes or head.get("ContentType") != item.mime_type:
raise DomainError("attachment_checksum_mismatch", 400, "Uploaded metadata does not match")
item.checksum_sha256 = checksum
item.completed_at = datetime.now(UTC)
session.add(audit("attachment.upload_completed", request_id, user.id, "attachment", item.id))
await session.commit()
return attachment_dto(item)
def attachment_dto(item: MessageAttachment) -> dict[str, Any]:
return {
"attachment_id": item.id,
"file_name": item.safe_file_name,
"mime_type": item.mime_type,
"size_bytes": item.size_bytes,
"checksum": f"sha256:{item.checksum_sha256}" if item.checksum_sha256 else None,
"scan_status": item.scan_status,
"completed_at": item.completed_at,
}
async def send_message(
session: AsyncSession,
user: UserIdentity,
dialog_id: uuid.UUID,
body: MessageRequest,
idem_key: str,
request_id: str,
settings: Settings,
safety: SafetyClient,
openlines: OpenLinesClient,
s3: S3Client,
fanout: RealtimeFanout,
) -> dict[str, Any]:
scope = f"dialogs.{dialog_id}.messages.create"
fingerprint = hashlib.sha256(
json.dumps(body.model_dump(mode="json"), sort_keys=True, separators=(",", ":")).encode()
).hexdigest()
idem = (
await session.execute(
select(IdempotencyRecord).where(
IdempotencyRecord.scope == scope,
IdempotencyRecord.user_id == user.id,
IdempotencyRecord.idempotency_key == idem_key,
)
)
).scalar_one_or_none()
if idem:
if idem.request_fingerprint != fingerprint:
raise DomainError("idempotency_key_reused", 409, "Idempotency key was reused")
if idem.status == "completed" and idem.response_body_json:
if idem.response_status == 422:
raise DomainError("message_blocked", 422, "Message was blocked by safety policy")
return idem.response_body_json
prior = (
await session.execute(
select(Message).where(
Message.dialog_id == dialog_id,
Message.client_idempotency_key == idem_key,
)
)
).scalar_one_or_none()
if prior and prior.delivery_status == "delivered":
return message_dto(prior)
raise DomainError(
"dependency_unavailable",
503,
"Previous request is still being recovered",
{"retry_after": 2},
)
dialog = await owned_dialog(session, user.id, dialog_id)
now, message_id = datetime.now(UTC), uuid.uuid4()
idem = IdempotencyRecord(
scope=scope,
user_id=user.id,
idempotency_key=idem_key,
request_fingerprint=fingerprint,
status="in_progress",
resource_type="message",
resource_id=message_id,
expires_at=now + timedelta(hours=24),
)
session.add(idem)
attachment: MessageAttachment | None = None
if isinstance(body, FileMessageRequest):
attachment = (
await session.execute(
select(MessageAttachment).where(
MessageAttachment.id == body.attachment_id,
MessageAttachment.owner_user_id == user.id,
MessageAttachment.dialog_id == dialog_id,
MessageAttachment.record_status == "A",
)
)
).scalar_one_or_none()
if not attachment or not attachment.completed_at:
raise DomainError("attachment_not_completed", 400, "Attachment upload is incomplete")
if attachment.checksum_sha256 != body.checksum.removeprefix("sha256:"):
raise DomainError("attachment_checksum_mismatch", 400, "Attachment checksum differs")
text, kind = "", "file"
else:
text, kind = unicodedata.normalize("NFKC", body.text).strip(), "text"
message = Message(
id=message_id,
dialog_id=dialog_id,
sender_type="client",
content_kind=kind,
text=text,
safety_status="pending",
delivery_status="accepted",
client_idempotency_key=idem_key,
occurred_at=now,
)
session.add(message)
if attachment:
attachment.message_id = message_id
session.add(audit("message.submitted", request_id, user.id, "message", message_id))
await session.commit()
await publish_message(fanout, message, settings, [attachment] if attachment else [])
payload: dict[str, Any] = {"message_id": str(message_id), "content_kind": kind, "text": text}
if attachment:
payload["attachment"] = {
"attachment_id": str(attachment.id),
"quarantine_object_key": attachment.quarantine_object_key,
"checksum": f"sha256:{attachment.checksum_sha256}",
"mime_type": attachment.mime_type,
"size_bytes": attachment.size_bytes,
}
try:
verdict = await safety.check(payload, request_id)
if verdict["_status"] == 203:
task_id = verdict["task_id"]
task = SafetyTask(
task_id=task_id,
message_id=message.id,
attachment_id=attachment.id if attachment else None,
quarantine_object_key=attachment.quarantine_object_key if attachment else None,
status="polling",
deadline_at=now
+ timedelta(seconds=settings.message_safety_task_poll_max_sec + 900),
next_poll_at=now,
)
session.add(task)
await session.commit()
deadline = time_monotonic() + settings.message_safety_task_poll_max_sec
while time_monotonic() < deadline:
await sleep(settings.message_safety_task_poll_interval_sec)
verdict = await safety.poll(task_id, request_id)
if verdict["_status"] != 203:
break
else:
raise DependencyFailure(timeout=True)
if verdict["_status"] == 403 or (
verdict["_status"] == 400
and verdict.get("error", {}).get("code") == "stub_final_error"
and verdict.get("verdict") == "deny"
):
message.text = ""
message.safety_status = "blocked"
message.delivery_status = "rejected"
if attachment and attachment.quarantine_object_key:
attachment.scan_status = "infected"
await s3.delete_quarantine(attachment.quarantine_object_key)
session.add(audit("message.blocked", request_id, user.id, "message", message.id))
idem.status = "completed"
idem.response_status = 422
idem.response_body_json = {
"error": {"code": "message_blocked", "message_id": str(message.id)}
}
await session.commit()
await publish_message_status(fanout, message, settings)
raise DomainError("message_blocked", 422, "Message was blocked by safety policy")
if verdict["_status"] != 200:
raise DependencyFailure()
if attachment and attachment.quarantine_object_key:
destination = f"attachments/dialogs/{dialog_id}/{attachment.id}"
await s3.promote(attachment.quarantine_object_key, destination)
attachment.storage_bucket = settings.selectel_s3_bucket_attachments
attachment.object_key = destination
attachment.quarantine_object_key = None
attachment.scan_status = "clean"
message.safety_status = "allowed"
outbox = DeliveryOutbox(
message_id=message.id,
external_chat_id=dialog_id,
payload_json={
"message_id": str(message.id),
"external_chat_id": str(dialog_id),
"occurred_at": message.occurred_at.isoformat(),
"user": {"id": str(user.id), "display_name": user.phone_number},
"message": {
"content_kind": message.content_kind,
"text": message.text,
"files": (
[
{
"attachment_id": str(attachment.id),
"name": attachment.safe_file_name,
"mime_type": attachment.mime_type,
"size_bytes": attachment.size_bytes,
"_storage_bucket": attachment.storage_bucket,
"_object_key": attachment.object_key,
}
]
if attachment
else []
),
},
},
next_attempt_at=datetime.now(UTC),
)
session.add(outbox)
await session.commit()
await publish_message_status(fanout, message, settings)
delivery_payload = await fresh_openlines_payload(outbox.payload_json, s3)
await openlines.send(message.id, delivery_payload, request_id)
message.delivery_status = "delivered"
dialog.status = "waiting_for_company"
dialog.last_message_at = datetime.now(UTC)
outbox.status = "delivered"
session.add(audit("message.delivered", request_id, user.id, "message", message.id))
result = message_dto(message, [attachment] if attachment else [])
idem.status = "completed"
idem.response_status = 201
idem.response_body_json = json.loads(json.dumps(result, default=str))
await session.commit()
await publish_message_status(fanout, message, settings)
await publish_dialog_status(fanout, dialog)
return result
except DomainError:
raise
except DependencyFailure as exc:
message.delivery_status = "failed"
session.add(audit("message.failed", request_id, user.id, "message", message.id))
await session.commit()
await publish_message_status(fanout, message, settings)
raise DomainError(
"dependency_timeout" if exc.timeout else "dependency_unavailable",
504 if exc.timeout else 503,
"A required dependency did not complete the request",
) from exc
async def apply_inbox(
session: AsyncSession,
event: OpenLinesInbox,
request_id: str,
snapshot: SettingsSnapshot,
s3: S3Client,
http: httpx.AsyncClient,
settings: Settings,
fanout: RealtimeFanout,
) -> tuple[dict[str, Any], int]:
fingerprint = hashlib.sha256(event.model_dump_json().encode()).hexdigest()
existing = (
await session.execute(
select(OpenLinesInboxReceipt).where(OpenLinesInboxReceipt.event_id == event.event_id)
)
).scalar_one_or_none()
if existing:
if existing.payload_fingerprint != fingerprint:
raise DomainError("idempotency_key_reused", 409, "Event id was reused")
return {"status": "duplicate"}, 200
dialog = (
await session.execute(
select(Dialog).where(Dialog.id == event.external_chat_id, Dialog.record_status == "A")
)
).scalar_one_or_none()
if not dialog:
raise DomainError("not_found", 404, "Resource was not found")
receipt = OpenLinesInboxReceipt(
event_id=event.event_id,
external_chat_id=event.external_chat_id,
bitrix_message_id=event.bitrix_message_id,
event_type=event.event_type,
payload_fingerprint=fingerprint,
status="processing",
)
session.add(receipt)
message: Message | None = None
attachment: MessageAttachment | None = None
if event.event_type == "dialog.closed":
dialog.status = "closed"
dialog.closed_at = event.occurred_at
else:
assert event.message is not None
inbound_file = event.message.files[0] if event.message.files else None
attachment_data: tuple[str, int, str] | None = None
if inbound_file:
extension = PurePath(inbound_file.name).suffix.lower().lstrip(".")
max_bytes = snapshot.integer("chat.attachments.max_size_mb") * 1024 * 1024
if (
extension not in snapshot.strings("chat.attachments.allowed_extensions")
or inbound_file.mime_type
not in snapshot.strings("chat.attachments.allowed_mime_types")
or inbound_file.size_bytes > max_bytes
):
raise DomainError("validation_error", 400, "Inbound file is not allowed")
attachment_id = uuid.uuid4()
object_key = f"attachments/dialogs/{dialog.id}/{attachment_id}"
try:
actual_size, checksum = await s3.upload_inbound(
http,
str(inbound_file.download_url),
object_key,
inbound_file.mime_type,
max_bytes,
)
except DependencyFailure as exc:
raise DomainError(
"dependency_unavailable", 503, "Inbound file transfer failed"
) from exc
if actual_size != inbound_file.size_bytes:
raise DomainError("validation_error", 400, "Inbound file size differs")
attachment_data = object_key, actual_size, checksum
message = Message(
dialog_id=dialog.id,
sender_type="company",
content_kind="file" if inbound_file else "text",
text=event.message.text,
safety_status="allowed",
delivery_status="delivered",
external_message_id=event.bitrix_message_id,
occurred_at=event.occurred_at,
)
session.add(message)
await session.flush()
if inbound_file and attachment_data:
object_key, actual_size, checksum = attachment_data
safe_name = re.sub(
r"[^A-Za-z0-9._-]",
"_",
unicodedata.normalize("NFKC", inbound_file.name),
)
attachment = MessageAttachment(
dialog_id=dialog.id,
message_id=message.id,
owner_user_id=dialog.user_id,
direction="company_inbound",
original_file_name=inbound_file.name,
safe_file_name=safe_name,
mime_type=inbound_file.mime_type,
size_bytes=actual_size,
checksum_sha256=checksum,
scan_status="clean",
storage_bucket=s3.settings.selectel_s3_bucket_attachments,
object_key=object_key,
completed_at=datetime.now(UTC),
)
session.add(attachment)
receipt.message_id = message.id
dialog.status = "waiting_for_client"
dialog.last_message_at = event.occurred_at
receipt.status = "applied"
session.add(audit("openlines.inbox_applied", request_id, dialog.user_id, "dialog", dialog.id))
await session.commit()
if message is not None:
await publish_message(fanout, message, settings, [attachment] if attachment else [])
await publish_dialog_status(fanout, dialog)
return {"status": "applied"}, 201
async def sleep(seconds: float) -> None:
import asyncio
await asyncio.sleep(seconds)
def time_monotonic() -> float:
import time
return time.monotonic()
@@ -0,0 +1,78 @@
from functools import lru_cache
from pydantic import AnyHttpUrl, Field, SecretStr
from pydantic_settings import BaseSettings, SettingsConfigDict
class Settings(BaseSettings):
model_config = SettingsConfigDict(
env_file=".env", env_file_encoding="utf-8", case_sensitive=True, extra="ignore"
)
app_env: str = Field(alias="APP_ENV")
api_port: int = Field(default=8000, alias="API_PORT")
log_level: str = Field(default="INFO", alias="LOG_LEVEL")
database_url: str = Field(alias="DATABASE_URL")
redis_url: str = Field(alias="REDIS_URL")
redis_realtime_url: str = Field(alias="REDIS_REALTIME_URL")
keycloak_public_url: AnyHttpUrl = Field(alias="KEYCLOAK_PUBLIC_URL")
keycloak_internal_url: AnyHttpUrl = Field(alias="KEYCLOAK_INTERNAL_URL")
keycloak_realm: str = Field(alias="KEYCLOAK_REALM")
keycloak_audience: str = Field(alias="KEYCLOAK_AUDIENCE")
jwks_cache_ttl_seconds: int = 300
jwks_stale_grace_seconds: int = 900
message_safety_url: AnyHttpUrl = Field(alias="MESSAGE_SAFETY_URL")
message_safety_service_token: SecretStr = Field(alias="MESSAGE_SAFETY_SERVICE_TOKEN")
message_safety_post_timeout_sec: float = Field(
default=5, alias="MESSAGE_SAFETY_POST_TIMEOUT_SEC"
)
message_safety_task_poll_interval_sec: float = Field(
default=2, alias="MESSAGE_SAFETY_TASK_POLL_INTERVAL_SEC"
)
message_safety_task_poll_max_sec: float = Field(
default=300, alias="MESSAGE_SAFETY_TASK_POLL_MAX_SEC"
)
message_safety_circuit_failure_threshold: int = Field(
default=5, alias="MESSAGE_SAFETY_CIRCUIT_FAILURE_THRESHOLD"
)
message_safety_circuit_open_sec: int = Field(
default=30, alias="MESSAGE_SAFETY_CIRCUIT_OPEN_SEC"
)
bitrix_local_app_base_url: AnyHttpUrl = Field(alias="BITRIX_LOCAL_APP_BASE_URL")
bitrix_local_app_internal_token: SecretStr = Field(alias="BITRIX_LOCAL_APP_INTERNAL_TOKEN")
bitrix_api_inbox_token: SecretStr = Field(alias="BITRIX_API_INBOX_TOKEN")
bitrix_local_app_http_timeout_sec: float = Field(
default=10, alias="BITRIX_LOCAL_APP_HTTP_TIMEOUT_SEC"
)
bitrix_local_app_circuit_failure_threshold: int = Field(
default=5, alias="BITRIX_LOCAL_APP_CIRCUIT_FAILURE_THRESHOLD"
)
bitrix_local_app_circuit_open_sec: int = Field(
default=30, alias="BITRIX_LOCAL_APP_CIRCUIT_OPEN_SEC"
)
keycloak_settings_bridge_token: SecretStr = Field(alias="KEYCLOAK_SETTINGS_BRIDGE_TOKEN")
selectel_s3_endpoint_url: AnyHttpUrl = Field(alias="SELECTEL_S3_ENDPOINT_URL")
selectel_s3_bucket_documents: str = Field(alias="SELECTEL_S3_BUCKET_DOCUMENTS")
selectel_s3_bucket_attachments: str = Field(alias="SELECTEL_S3_BUCKET_ATTACHMENTS")
selectel_s3_bucket_quarantine: str = Field(alias="SELECTEL_S3_BUCKET_QUARANTINE")
selectel_s3_access_key: SecretStr = Field(alias="SELECTEL_S3_ACCESS_KEY")
selectel_s3_secret_key: SecretStr = Field(alias="SELECTEL_S3_SECRET_KEY")
otel_exporter_otlp_endpoint: str | None = Field(
default=None, alias="OTEL_EXPORTER_OTLP_ENDPOINT"
)
cursor_hmac_secret: SecretStr = Field(alias="CURSOR_HMAC_SECRET")
trusted_proxy_cidrs: str = Field(default="127.0.0.1/32", alias="TRUSTED_PROXY_CIDRS")
worker_poll_interval_sec: float = Field(default=2, alias="WORKER_POLL_INTERVAL_SEC")
@property
def issuer(self) -> str:
return f"{str(self.keycloak_public_url).rstrip('/')}/realms/{self.keycloak_realm}"
@lru_cache
def get_settings() -> Settings:
return Settings()
+234
View File
@@ -0,0 +1,234 @@
import asyncio
import uuid
from datetime import UTC, datetime, timedelta
import httpx
import redis.asyncio as redis
import structlog
from sqlalchemy import select
from app.db import Database, DeliveryOutbox, Dialog, Message, MessageAttachment, SafetyTask
from app.integrations import (
DependencyFailure,
OpenLinesClient,
S3Client,
SafetyClient,
fresh_openlines_payload,
)
from app.realtime import RealtimeFanout
from app.services import publish_dialog_status, publish_message_status
from app.settings import Settings, get_settings
log = structlog.get_logger()
async def delivery_once(
db: Database,
client: OpenLinesClient,
s3: S3Client,
fanout: RealtimeFanout,
settings: Settings,
worker_id: str,
batch_size: int = 20,
) -> int:
async with db.sessions() as session:
rows = (
(
await session.execute(
select(DeliveryOutbox)
.where(
DeliveryOutbox.status.in_(["pending", "retry"]),
DeliveryOutbox.next_attempt_at <= datetime.now(UTC),
)
.with_for_update(skip_locked=True)
.limit(batch_size)
)
)
.scalars()
.all()
)
ids = [row.id for row in rows]
for row in rows:
row.status = "processing"
row.locked_at = datetime.now(UTC)
row.locked_by = worker_id
await session.commit()
for row_id in ids:
async with db.sessions() as session:
row = await session.get(DeliveryOutbox, row_id, with_for_update=True)
if row is None:
continue
message = None
dialog = None
try:
payload = await fresh_openlines_payload(row.payload_json, s3)
await client.send(row.message_id, payload, f"worker-{worker_id}")
row.status = "delivered"
message = await session.get(Message, row.message_id)
if message:
message.delivery_status = "delivered"
dialog = await session.get(Dialog, message.dialog_id)
if dialog:
dialog.status = "waiting_for_company"
dialog.last_message_at = datetime.now(UTC)
except DependencyFailure:
row.attempt_count += 1
row.status = "dead_letter" if row.attempt_count >= 12 else "retry"
row.next_attempt_at = datetime.now(UTC) + timedelta(
seconds=min(3600, 2**row.attempt_count)
)
row.last_error_code = "dependency_unavailable"
row.locked_at = None
row.locked_by = None
await session.commit()
if message:
await publish_message_status(fanout, message, settings)
if dialog:
await publish_dialog_status(fanout, dialog)
return len(ids)
async def safety_once(
db: Database,
safety: SafetyClient,
s3: S3Client,
fanout: RealtimeFanout,
settings: Settings,
worker_id: str,
batch_size: int = 20,
) -> int:
async with db.sessions() as session:
rows = (
(
await session.execute(
select(SafetyTask)
.where(
SafetyTask.status.in_(["polling", "failed"]),
SafetyTask.next_poll_at <= datetime.now(UTC),
SafetyTask.deadline_at > datetime.now(UTC),
)
.with_for_update(skip_locked=True)
.limit(batch_size)
)
)
.scalars()
.all()
)
ids = [row.id for row in rows]
for row in rows:
row.locked_at, row.locked_by = datetime.now(UTC), worker_id
await session.commit()
for task_id in ids:
async with db.sessions() as session:
task = await session.get(SafetyTask, task_id, with_for_update=True)
if task is None:
continue
message = None
try:
verdict = await safety.poll(task.task_id, f"worker-{worker_id}")
message = await session.get(Message, task.message_id)
attachment = (
await session.get(MessageAttachment, task.attachment_id)
if task.attachment_id
else None
)
if verdict["_status"] == 200 and message:
if attachment and attachment.quarantine_object_key:
destination = f"attachments/dialogs/{message.dialog_id}/{attachment.id}"
await s3.promote(attachment.quarantine_object_key, destination)
attachment.storage_bucket = s3.settings.selectel_s3_bucket_attachments
attachment.object_key = destination
attachment.quarantine_object_key = None
attachment.scan_status = "clean"
message.safety_status = "allowed"
task.status = "completed"
elif verdict["_status"] == 403 and message:
message.text = ""
message.safety_status = "blocked"
message.delivery_status = "rejected"
if attachment and attachment.quarantine_object_key:
await s3.delete_quarantine(attachment.quarantine_object_key)
attachment.scan_status = "infected"
task.status = "completed"
else:
task.next_poll_at = datetime.now(UTC) + timedelta(seconds=2)
except DependencyFailure:
task.attempt_count += 1
task.status = "failed"
task.next_poll_at = datetime.now(UTC) + timedelta(
seconds=min(300, 2**task.attempt_count)
)
task.locked_at = None
task.locked_by = None
await session.commit()
if message:
await publish_message_status(fanout, message, settings)
return len(ids)
async def cleanup_once(db: Database, s3: S3Client, batch_size: int = 100) -> int:
async with db.sessions() as session:
rows = (
(
await session.execute(
select(MessageAttachment)
.where(
MessageAttachment.record_status == "A",
MessageAttachment.quarantine_object_key.is_not(None),
MessageAttachment.upload_expires_at < datetime.now(UTC),
MessageAttachment.message_id.is_(None),
)
.with_for_update(skip_locked=True)
.limit(batch_size)
)
)
.scalars()
.all()
)
for row in rows:
if row.quarantine_object_key:
await s3.delete_quarantine(row.quarantine_object_key)
row.record_status = "D"
row.status_changed_at = datetime.now(UTC)
row.status_change_reason = "expired_quarantine_cleanup"
await session.commit()
return len(rows)
async def loop(kind: str) -> None:
settings = get_settings()
db = Database(settings.database_url)
http = httpx.AsyncClient()
safety = SafetyClient(settings, http)
openlines = OpenLinesClient(settings, http)
s3 = S3Client(settings)
redis_rt = redis.from_url(settings.redis_realtime_url, decode_responses=True)
fanout = RealtimeFanout(redis_rt)
worker_id = f"{kind}-{uuid.uuid4()}"
try:
while True:
count = 0
if kind == "delivery":
count = await delivery_once(db, openlines, s3, fanout, settings, worker_id)
elif kind == "safety":
count = await safety_once(db, safety, s3, fanout, settings, worker_id)
else:
count = await cleanup_once(db, s3)
if not count:
await asyncio.sleep(settings.worker_poll_interval_sec)
finally:
await http.aclose()
await redis_rt.aclose()
await db.close()
def delivery_main() -> None:
asyncio.run(loop("delivery"))
def safety_main() -> None:
asyncio.run(loop("safety"))
def cleanup_main() -> None:
asyncio.run(loop("cleanup"))
@@ -0,0 +1,56 @@
services:
api-backend:
build: .
image: han-chat/api-backend:local
expose:
- "8000"
environment:
APP_ENV: ${APP_ENV}
API_PORT: ${API_PORT:-8000}
LOG_LEVEL: ${LOG_LEVEL:-INFO}
DATABASE_URL: ${DATABASE_URL}
REDIS_URL: ${REDIS_URL}
REDIS_REALTIME_URL: ${REDIS_REALTIME_URL}
KEYCLOAK_PUBLIC_URL: ${KEYCLOAK_PUBLIC_URL}
KEYCLOAK_INTERNAL_URL: ${KEYCLOAK_INTERNAL_URL}
KEYCLOAK_REALM: ${KEYCLOAK_REALM}
KEYCLOAK_AUDIENCE: ${KEYCLOAK_AUDIENCE}
MESSAGE_SAFETY_URL: ${MESSAGE_SAFETY_URL}
MESSAGE_SAFETY_SERVICE_TOKEN: ${MESSAGE_SAFETY_SERVICE_TOKEN}
MESSAGE_SAFETY_POST_TIMEOUT_SEC: ${MESSAGE_SAFETY_POST_TIMEOUT_SEC:-5}
MESSAGE_SAFETY_TASK_POLL_INTERVAL_SEC: ${MESSAGE_SAFETY_TASK_POLL_INTERVAL_SEC:-2}
MESSAGE_SAFETY_TASK_POLL_MAX_SEC: ${MESSAGE_SAFETY_TASK_POLL_MAX_SEC:-300}
MESSAGE_SAFETY_CIRCUIT_FAILURE_THRESHOLD: ${MESSAGE_SAFETY_CIRCUIT_FAILURE_THRESHOLD:-5}
MESSAGE_SAFETY_CIRCUIT_OPEN_SEC: ${MESSAGE_SAFETY_CIRCUIT_OPEN_SEC:-30}
BITRIX_LOCAL_APP_BASE_URL: ${BITRIX_LOCAL_APP_BASE_URL}
BITRIX_LOCAL_APP_INTERNAL_TOKEN: ${BITRIX_LOCAL_APP_INTERNAL_TOKEN}
BITRIX_API_INBOX_TOKEN: ${BITRIX_API_INBOX_TOKEN}
BITRIX_LOCAL_APP_HTTP_TIMEOUT_SEC: ${BITRIX_LOCAL_APP_HTTP_TIMEOUT_SEC:-10}
BITRIX_LOCAL_APP_CIRCUIT_FAILURE_THRESHOLD: ${BITRIX_LOCAL_APP_CIRCUIT_FAILURE_THRESHOLD:-5}
BITRIX_LOCAL_APP_CIRCUIT_OPEN_SEC: ${BITRIX_LOCAL_APP_CIRCUIT_OPEN_SEC:-30}
KEYCLOAK_SETTINGS_BRIDGE_TOKEN: ${KEYCLOAK_SETTINGS_BRIDGE_TOKEN}
SELECTEL_S3_ENDPOINT_URL: ${SELECTEL_S3_ENDPOINT_URL}
SELECTEL_S3_BUCKET_DOCUMENTS: ${SELECTEL_S3_BUCKET_DOCUMENTS}
SELECTEL_S3_BUCKET_ATTACHMENTS: ${SELECTEL_S3_BUCKET_ATTACHMENTS}
SELECTEL_S3_BUCKET_QUARANTINE: ${SELECTEL_S3_BUCKET_QUARANTINE}
SELECTEL_S3_ACCESS_KEY: ${SELECTEL_S3_ACCESS_KEY}
SELECTEL_S3_SECRET_KEY: ${SELECTEL_S3_SECRET_KEY}
OTEL_EXPORTER_OTLP_ENDPOINT: ${OTEL_EXPORTER_OTLP_ENDPOINT}
CURSOR_HMAC_SECRET: ${CURSOR_HMAC_SECRET}
healthcheck:
test:
- CMD
- python
- -c
- "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8000/health/live', timeout=2)"
interval: 10s
timeout: 3s
retries: 5
restart: unless-stopped
networks:
- backend
- observability
networks:
backend:
observability:
+314
View File
@@ -0,0 +1,314 @@
openapi: 3.1.0
info:
title: HAN Chat API
version: 1.0.0
servers:
- url: /
paths:
/health/live:
get:
operationId: healthLive
responses:
"200": {description: Process is live}
/health/ready:
get:
operationId: healthReady
responses:
"200": {description: Ready or partially degraded}
"503": {$ref: "#/components/responses/DependencyUnavailable"}
/api/v1/public/app-config:
get:
operationId: getPublicAppConfig
responses:
"200": {description: Public application configuration}
/api/v1/public/content:
get:
operationId: getPublicContent
parameters:
- {name: locale, in: query, schema: {type: string, default: ru}}
responses:
"200": {description: Active UI content}
/api/v1/auth/bootstrap:
post:
operationId: bootstrap
security: [{bearerAuth: []}]
requestBody:
required: true
content:
application/json:
schema: {$ref: "#/components/schemas/BootstrapRequest"}
responses:
"200": {description: Local user resolved}
"401": {$ref: "#/components/responses/Unauthorized"}
/api/v1/consents:
post:
operationId: recordConsents
security: [{bearerAuth: []}]
requestBody:
required: true
content:
application/json:
schema: {$ref: "#/components/schemas/ConsentsRequest"}
responses:
"201": {description: Immutable consent records saved}
/api/v1/analytics/session-start:
post:
operationId: startUxSession
security: [{bearerAuth: []}]
requestBody:
required: true
content:
application/json:
schema: {$ref: "#/components/schemas/SessionStartRequest"}
responses:
"201": {description: UX session created}
/api/v1/me:
get:
operationId: getCurrentProfile
security: [{bearerAuth: []}]
responses:
"200": {description: Block profile}
/api/v1/me/documents:
get:
operationId: listDocuments
security: [{bearerAuth: []}]
responses:
"200": {description: Cursor-paginated documents}
/api/v1/documents/{document_id}:
parameters:
- {$ref: "#/components/parameters/DocumentId"}
get:
operationId: getDocument
security: [{bearerAuth: []}]
responses:
"200": {description: Document metadata}
"404": {$ref: "#/components/responses/NotFound"}
/api/v1/documents/{document_id}/download-url:
parameters:
- {$ref: "#/components/parameters/DocumentId"}
get:
operationId: getDocumentDownloadUrl
security: [{bearerAuth: []}]
responses:
"200": {description: Short-lived presigned GET}
/api/v1/dialogs:
post:
operationId: createDialog
security: [{bearerAuth: []}]
parameters:
- {$ref: "#/components/parameters/IdempotencyKey"}
responses:
"200": {description: Existing active dialog}
"201": {description: New active dialog}
get:
operationId: listDialogs
security: [{bearerAuth: []}]
responses:
"200": {description: Cursor-paginated dialogs}
/api/v1/dialogs/{dialog_id}:
parameters:
- {$ref: "#/components/parameters/DialogId"}
get:
operationId: getDialog
security: [{bearerAuth: []}]
responses:
"200": {description: Dialog summary}
"404": {$ref: "#/components/responses/NotFound"}
/api/v1/dialogs/{dialog_id}/messages:
parameters:
- {$ref: "#/components/parameters/DialogId"}
get:
operationId: listMessages
security: [{bearerAuth: []}]
parameters:
- {name: after, in: query, schema: {type: string}}
- {name: limit, in: query, schema: {type: integer, minimum: 1, maximum: 100, default: 50}}
responses:
"200": {description: Message history or polling delta}
post:
operationId: sendMessage
security: [{bearerAuth: []}]
parameters:
- {$ref: "#/components/parameters/IdempotencyKey"}
requestBody:
required: true
content:
application/json:
schema:
oneOf:
- {$ref: "#/components/schemas/TextMessageRequest"}
- {$ref: "#/components/schemas/FileMessageRequest"}
discriminator: {propertyName: content_kind}
responses:
"201": {description: Safety-allowed and delivered message}
"422": {description: Message blocked, content redacted}
"503": {$ref: "#/components/responses/DependencyUnavailable"}
/api/v1/dialogs/{dialog_id}/attachments/init:
parameters:
- {$ref: "#/components/parameters/DialogId"}
post:
operationId: initAttachment
security: [{bearerAuth: []}]
requestBody:
required: true
content:
application/json:
schema: {$ref: "#/components/schemas/AttachmentInitRequest"}
responses:
"201": {description: Upload metadata and presigned PUT}
/api/v1/dialogs/{dialog_id}/attachments/{attachment_id}/complete:
parameters:
- {$ref: "#/components/parameters/DialogId"}
- {$ref: "#/components/parameters/AttachmentId"}
post:
operationId: completeAttachment
security: [{bearerAuth: []}]
requestBody:
required: true
content:
application/json:
schema: {$ref: "#/components/schemas/ChecksumRequest"}
responses:
"200": {description: Upload metadata verified}
/api/v1/dialogs/{dialog_id}/attachments/{attachment_id}/download-url:
parameters:
- {$ref: "#/components/parameters/DialogId"}
- {$ref: "#/components/parameters/AttachmentId"}
get:
operationId: getAttachmentDownloadUrl
security: [{bearerAuth: []}]
responses:
"200": {description: Audited short-lived presigned GET}
/internal/openlines/v1/inbox:
post:
operationId: applyOpenLinesInbox
security: [{serviceBearer: []}]
requestBody:
required: true
content:
application/json:
schema: {$ref: "#/components/schemas/OpenLinesInbox"}
responses:
"200": {description: Duplicate acknowledged}
"201": {description: Event applied}
/internal/settings/v1/otp:
get:
operationId: getOtpSettings
security: [{serviceBearer: []}]
responses:
"200": {description: Product OTP limits and cache metadata}
"503": {$ref: "#/components/responses/DependencyUnavailable"}
components:
securitySchemes:
bearerAuth: {type: http, scheme: bearer, bearerFormat: JWT}
serviceBearer: {type: http, scheme: bearer}
parameters:
DialogId: {name: dialog_id, in: path, required: true, schema: {type: string, format: uuid}}
AttachmentId: {name: attachment_id, in: path, required: true, schema: {type: string, format: uuid}}
DocumentId: {name: document_id, in: path, required: true, schema: {type: string, format: uuid}}
IdempotencyKey: {name: Idempotency-Key, in: header, required: true, schema: {type: string, minLength: 1, maxLength: 128}}
responses:
Unauthorized:
description: JWT is absent, invalid or expired
content: {application/json: {schema: {$ref: "#/components/schemas/ErrorEnvelope"}}}
NotFound:
description: Resource absent or owned by another user
content: {application/json: {schema: {$ref: "#/components/schemas/ErrorEnvelope"}}}
DependencyUnavailable:
description: Required dependency is unavailable
content: {application/json: {schema: {$ref: "#/components/schemas/ErrorEnvelope"}}}
schemas:
ErrorEnvelope:
type: object
required: [error]
properties:
error:
type: object
required: [code, message, request_id, details]
properties:
code: {type: string}
message: {type: string}
request_id: {type: string}
details: {type: object}
ConsentChoice:
type: object
additionalProperties: false
required: [accepted, version]
properties:
accepted: {type: boolean}
version: {type: string, maxLength: 64}
ConsentSet:
type: object
additionalProperties: false
required: [personal_data, user_agreement, marketing]
properties:
personal_data: {$ref: "#/components/schemas/ConsentChoice"}
user_agreement: {$ref: "#/components/schemas/ConsentChoice"}
marketing: {$ref: "#/components/schemas/ConsentChoice"}
Device:
type: object
additionalProperties: false
required: [platform, app_version]
properties:
platform: {type: string, enum: [ios, android, web]}
app_version: {type: string, maxLength: 64}
device_id: {type: [string, "null"], maxLength: 255}
BootstrapRequest:
type: object
additionalProperties: false
required: [consents, device]
properties:
consents: {$ref: "#/components/schemas/ConsentSet"}
device: {$ref: "#/components/schemas/Device"}
ConsentsRequest:
type: object
additionalProperties: false
required: [consents]
properties: {consents: {$ref: "#/components/schemas/ConsentSet"}}
SessionStartRequest:
type: object
additionalProperties: false
required: [start_reason, device]
properties:
start_reason: {type: string, enum: [first_launch, cold_start, idle_timeout]}
device: {$ref: "#/components/schemas/Device"}
TextMessageRequest:
type: object
additionalProperties: false
required: [content_kind, text]
properties:
content_kind: {const: text}
text: {type: string, minLength: 1, maxLength: 4000}
FileMessageRequest:
type: object
additionalProperties: false
required: [content_kind, attachment_id, checksum]
properties:
content_kind: {const: file}
attachment_id: {type: string, format: uuid}
checksum: {type: string, pattern: "^sha256:[0-9a-f]{64}$"}
AttachmentInitRequest:
type: object
additionalProperties: false
required: [file_name, mime_type, size_bytes]
properties:
file_name: {type: string, minLength: 1, maxLength: 255}
mime_type: {type: string, minLength: 1, maxLength: 128}
size_bytes: {type: integer, minimum: 1}
ChecksumRequest:
type: object
additionalProperties: false
required: [checksum]
properties:
checksum: {type: string, pattern: "^sha256:[0-9a-f]{64}$"}
OpenLinesInbox:
type: object
additionalProperties: false
required: [event_id, event_type, external_chat_id, occurred_at]
properties:
event_id: {type: string, minLength: 1, maxLength: 255}
event_type: {type: string, enum: [message.new, dialog.closed]}
external_chat_id: {type: string, format: uuid}
bitrix_message_id: {type: [string, "null"], maxLength: 255}
occurred_at: {type: string, format: date-time}
message: {type: [object, "null"]}
@@ -0,0 +1,67 @@
[project]
name = "han-api-backend"
version = "0.1.0"
description = "HAN Chat FastAPI backend"
requires-python = ">=3.12"
dependencies = [
"alembic>=1.16,<2",
"asyncpg>=0.30,<1",
"boto3>=1.39,<2",
"fastapi>=0.116,<1",
"httpx>=0.28,<1",
"phonenumbers>=9,<10",
"prometheus-client>=0.22,<1",
"pydantic-settings>=2.10,<3",
"pyyaml>=6,<7",
"pyjwt[crypto]>=2.10,<3",
"redis>=6,<7",
"sqlalchemy[asyncio]>=2.0.41,<3",
"structlog>=25,<26",
"uvicorn[standard]>=0.35,<1",
]
[project.optional-dependencies]
dev = [
"aiosqlite>=0.21,<1",
"mypy>=1.16,<2",
"pytest>=8.4,<9",
"pytest-asyncio>=1.0,<2",
"ruff>=0.12,<1",
"types-boto3-s3>=1.39,<2",
"types-PyYAML>=6,<7",
]
[project.scripts]
han-api = "app.main:run"
han-delivery-worker = "app.workers:delivery_main"
han-safety-worker = "app.workers:safety_main"
han-cleanup-worker = "app.workers:cleanup_main"
[build-system]
requires = ["hatchling"]
build-backend = "hatchling.build"
[tool.hatch.build.targets.wheel]
packages = ["app"]
[tool.pytest.ini_options]
asyncio_mode = "auto"
testpaths = ["tests"]
[tool.ruff]
target-version = "py312"
line-length = 100
[tool.ruff.lint]
select = ["E", "F", "I", "UP", "B", "ASYNC", "S"]
ignore = ["S101"]
[tool.mypy]
python_version = "3.12"
check_untyped_defs = true
warn_redundant_casts = true
warn_unused_ignores = true
ignore_missing_imports = true
disable_error_code = ["no-untyped-def", "misc", "arg-type", "assignment"]
plugins = ["pydantic.mypy", "sqlalchemy.ext.mypy.plugin"]
exclude = ["alembic/"]
@@ -0,0 +1,155 @@
import uuid
import httpx
import pytest
from app.integrations import (
DependencyFailure,
OpenLinesClient,
SafetyClient,
fresh_openlines_payload,
)
from app.settings import Settings
def settings() -> Settings:
common = {
"APP_ENV": "test",
"DATABASE_URL": "postgresql+asyncpg://u:p@localhost/db",
"REDIS_URL": "redis://localhost/0",
"REDIS_REALTIME_URL": "redis://localhost/1",
"KEYCLOAK_PUBLIC_URL": "https://auth.example",
"KEYCLOAK_INTERNAL_URL": "http://keycloak:8080",
"KEYCLOAK_REALM": "han",
"KEYCLOAK_AUDIENCE": "api",
"MESSAGE_SAFETY_URL": "http://safety:8080",
"MESSAGE_SAFETY_SERVICE_TOKEN": "safety-token",
"BITRIX_LOCAL_APP_BASE_URL": "http://bitrix:8080",
"BITRIX_LOCAL_APP_INTERNAL_TOKEN": "bitrix-token",
"BITRIX_API_INBOX_TOKEN": "inbox-token",
"KEYCLOAK_SETTINGS_BRIDGE_TOKEN": "settings-token",
"SELECTEL_S3_ENDPOINT_URL": "https://s3.example",
"SELECTEL_S3_BUCKET_DOCUMENTS": "documents",
"SELECTEL_S3_BUCKET_ATTACHMENTS": "attachments",
"SELECTEL_S3_BUCKET_QUARANTINE": "quarantine",
"SELECTEL_S3_ACCESS_KEY": "access",
"SELECTEL_S3_SECRET_KEY": "secret",
"CURSOR_HMAC_SECRET": "x" * 32,
}
return Settings.model_validate(common)
@pytest.mark.asyncio
async def test_safety_contract_status_and_service_token() -> None:
async def handler(request: httpx.Request) -> httpx.Response:
assert request.headers["X-Service-Token"] == "safety-token"
assert request.url.path == "/internal/safety/v1/messages/check"
return httpx.Response(203, json={"verdict": "pending", "task_id": "task-1"})
async with httpx.AsyncClient(transport=httpx.MockTransport(handler)) as http:
result = await SafetyClient(settings(), http).check(
{"message_id": str(uuid.uuid4()), "content_kind": "text", "text": "hello"},
"request-1",
)
assert result == {"verdict": "pending", "task_id": "task-1", "_status": 203}
@pytest.mark.asyncio
async def test_safety_file_body_uses_exact_attachment_schema() -> None:
attachment_id = uuid.uuid4()
async def handler(request: httpx.Request) -> httpx.Response:
body = __import__("json").loads(request.content)
assert body["attachment"] == {
"attachment_id": str(attachment_id),
"quarantine_object_key": "quarantine/users/u/file",
"mime_type": "application/pdf",
"size_bytes": 42,
"checksum": "sha256:" + "a" * 64,
}
assert "file" not in body
return httpx.Response(200, json={"verdict": "allow"})
payload = {
"message_id": str(uuid.uuid4()),
"content_kind": "file",
"text": "",
"attachment": {
"attachment_id": str(attachment_id),
"quarantine_object_key": "quarantine/users/u/file",
"mime_type": "application/pdf",
"size_bytes": 42,
"checksum": "sha256:" + "a" * 64,
},
}
async with httpx.AsyncClient(transport=httpx.MockTransport(handler)) as http:
await SafetyClient(settings(), http).check(payload, "request-1")
@pytest.mark.asyncio
async def test_safety_auth_failure_is_dependency_failure() -> None:
async def handler(_: httpx.Request) -> httpx.Response:
return httpx.Response(401, json={"error": {"code": "service_unauthorized"}})
async with httpx.AsyncClient(transport=httpx.MockTransport(handler)) as http:
with pytest.raises(DependencyFailure):
await SafetyClient(settings(), http).check(
{"message_id": str(uuid.uuid4()), "content_kind": "text", "text": "hello"},
"request-1",
)
@pytest.mark.asyncio
async def test_openlines_contract_uses_bearer_and_idempotency() -> None:
message_id = uuid.uuid4()
async def handler(request: httpx.Request) -> httpx.Response:
assert request.headers["Authorization"] == "Bearer bitrix-token"
assert request.headers["Idempotency-Key"] == str(message_id)
assert request.url.path == "/internal/openlines/v1/messages"
return httpx.Response(201, json={"status": "accepted"})
async with httpx.AsyncClient(transport=httpx.MockTransport(handler)) as http:
result = await OpenLinesClient(settings(), http).send(
message_id, {"message_id": str(message_id)}, "request-1"
)
assert result["status"] == "accepted"
@pytest.mark.asyncio
async def test_openlines_payload_gets_fresh_download_url_without_storage_fields() -> None:
class FakeS3:
calls = 0
async def presign_get(self, bucket: str, key: str, ttl: int = 300) -> str:
self.calls += 1
return f"https://download.example/{bucket}/{key}?generation={self.calls}"
payload = {
"message_id": str(uuid.uuid4()),
"external_chat_id": str(uuid.uuid4()),
"occurred_at": "2026-07-10T12:00:00+00:00",
"user": {"id": str(uuid.uuid4()), "display_name": "+79990000000"},
"message": {
"content_kind": "file",
"text": "",
"files": [{
"attachment_id": str(uuid.uuid4()),
"name": "file.pdf",
"mime_type": "application/pdf",
"size_bytes": 42,
"_storage_bucket": "attachments",
"_object_key": "dialogs/d/file",
}],
},
}
s3 = FakeS3()
first = await fresh_openlines_payload(payload, s3) # type: ignore[arg-type]
second = await fresh_openlines_payload(payload, s3) # type: ignore[arg-type]
assert first["occurred_at"] and first["user"]["id"]
assert first["message"]["content_kind"] == "file"
assert (
first["message"]["files"][0]["download_url"]
!= second["message"]["files"][0]["download_url"]
)
assert all(not key.startswith("_") for key in first["message"]["files"][0])
@@ -0,0 +1,58 @@
import base64
from pathlib import Path
from types import SimpleNamespace
import yaml
from app.main import app, websocket_token
EXPECTED_PATHS = {
"/health/live",
"/health/ready",
"/api/v1/public/app-config",
"/api/v1/public/content",
"/api/v1/auth/bootstrap",
"/api/v1/consents",
"/api/v1/analytics/session-start",
"/api/v1/me",
"/api/v1/me/documents",
"/api/v1/documents/{document_id}",
"/api/v1/documents/{document_id}/download-url",
"/api/v1/dialogs",
"/api/v1/dialogs/{dialog_id}",
"/api/v1/dialogs/{dialog_id}/messages",
"/api/v1/dialogs/{dialog_id}/attachments/init",
"/api/v1/dialogs/{dialog_id}/attachments/{attachment_id}/complete",
"/api/v1/dialogs/{dialog_id}/attachments/{attachment_id}/download-url",
"/internal/openlines/v1/inbox",
"/internal/settings/v1/otp",
}
def test_openapi_31_contains_all_http_contracts() -> None:
schema = app.openapi()
assert schema["openapi"].startswith("3.1.")
assert EXPECTED_PATHS <= schema["paths"].keys()
assert all(not path.startswith("/internal/safety") for path in schema["paths"])
committed = yaml.safe_load(Path("openapi.yaml").read_text(encoding="utf-8"))
assert committed["openapi"] == "3.1.0"
assert committed["paths"].keys() == schema["paths"].keys()
def test_websocket_route_is_registered() -> None:
assert any(getattr(route, "path", None) == "/api/v1/realtime" for route in app.routes)
def test_websocket_accepts_canonical_base64url_jwt_protocol() -> None:
jwt = "header.payload.signature"
encoded = base64.urlsafe_b64encode(jwt.encode()).decode().rstrip("=")
websocket = SimpleNamespace(
headers={"sec-websocket-protocol": f"han-chat-v1, han.jwt.{encoded}"},
query_params={},
)
assert websocket_token(websocket) == (jwt, f"han.jwt.{encoded}")
def test_committed_openapi_server_does_not_double_api_prefix() -> None:
committed = yaml.safe_load(Path("openapi.yaml").read_text(encoding="utf-8"))
assert committed["servers"] == [{"url": "/"}]
@@ -0,0 +1,26 @@
from pathlib import Path
import pytest
from app.cli.seed_settings import load_seed
from app.services import REQUIRED_SETTINGS
def test_production_like_seed_contains_all_mandatory_settings() -> None:
path = Path(__file__).resolve().parents[3] / "deployment/app-settings.production-like.yaml"
rows = load_seed(path)
assert REQUIRED_SETTINGS <= {row["setting_key"] for row in rows}
assert all(row["record_status"] == "A" for row in rows)
def test_seed_rejects_invalid_typed_value(tmp_path: Path) -> None:
path = tmp_path / "settings.yaml"
path.write_text(
"schema_version: 1\nsettings:\n"
" bad.integer: {type: integer, value: nope, public: false}\n",
encoding="utf-8",
)
with pytest.raises(ValueError, match="integer value expected"):
load_seed(path)
@@ -0,0 +1,80 @@
import uuid
import pytest
from pydantic import TypeAdapter, ValidationError
from app.auth import canonical_phone
from app.integrations import CircuitBreaker, RateLimiter
from app.schemas import (
FileMessageRequest,
MessageRequest,
TextMessageRequest,
canonical_fingerprint,
decode_cursor,
encode_cursor,
)
def test_phone_claim_priority_and_e164_validation() -> None:
claims = {"phone_number": "+74999591007", "preferred_username": "+12025550123"}
assert canonical_phone(claims) == "+74999591007"
assert canonical_phone({"phone_number": "89999999999"}) is None
def test_message_discriminated_union() -> None:
adapter = TypeAdapter(MessageRequest)
assert isinstance(
adapter.validate_python({"content_kind": "text", "text": "Здравствуйте"}),
TextMessageRequest,
)
assert isinstance(
adapter.validate_python(
{
"content_kind": "file",
"attachment_id": str(uuid.uuid4()),
"checksum": "sha256:" + "a" * 64,
}
),
FileMessageRequest,
)
with pytest.raises(ValidationError):
adapter.validate_python(
{"content_kind": "text", "text": "", "attachment_id": str(uuid.uuid4())}
)
def test_fingerprint_is_canonical_and_user_scoped() -> None:
user = uuid.uuid4()
first = canonical_fingerprint("post", "/dialogs/{id}", {"id": "1"}, {"b": 2, "a": 1}, user)
second = canonical_fingerprint("POST", "/dialogs/{id}", {"id": "1"}, {"a": 1, "b": 2}, user)
assert first == second
assert first != canonical_fingerprint(
"POST", "/dialogs/{id}", {"id": "1"}, {"a": 1, "b": 2}, uuid.uuid4()
)
def test_cursor_roundtrip_and_tamper_rejection() -> None:
secret = b"test-secret" * 4
cursor = encode_cursor({"id": str(uuid.uuid4()), "created_at": "2026-01-01"}, secret)
assert decode_cursor(cursor, secret)["created_at"] == "2026-01-01"
with pytest.raises(ValueError, match="invalid cursor"):
decode_cursor(cursor[:-2] + "aa", secret)
def test_rate_limit_keys_do_not_expose_identity() -> None:
key = RateLimiter.key("ip", "203.0.113.7", "public", 60)
assert "203.0.113.7" not in key
assert key.startswith("han:api:rl:ip:")
def test_circuit_breaker_opens_and_half_opens(monkeypatch: pytest.MonkeyPatch) -> None:
clock = [10.0]
monkeypatch.setattr("app.integrations.time.monotonic", lambda: clock[0])
breaker = CircuitBreaker(2, 30)
breaker.failure()
breaker.failure()
assert not breaker.allow()
clock[0] = 41.0
assert breaker.allow()
breaker.success()
assert breaker.allow()