Admin/han-app
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Проект разделен на два репозитория

Browse Source
This commit is contained in:
mi
2026-08-14 15:42:45 +03:00
parent e06a77ee1d
commit bbef7a30c9
521 changed files with 2597 additions and 2302 deletions
Download Patch File Download Diff File Expand all files Collapse all files
VM1_app/codebase/backend/.env.example
+131
View File
@@ -0,0 +1,131 @@
# Non-secret deployment configuration only.
# Copy to .env, replace placeholders, then run: ./scripts/validate-env .env
# Secrets are supplied at runtime by deployment/secrets/han-secrets.
SECRETS_SOURCE=file
APP_ENV=production-like
RELEASE_VERSION=change-me-release
LOG_LEVEL=INFO
COMPOSE_PROJECT_NAME=han-chat
# Immutable service images built by their owning modules.
API_BACKEND_IMAGE=han-chat-api-backend:local
MESSAGE_SAFETY_IMAGE=han-chat-message-safety:local
BITRIX_LOCAL_APP_IMAGE=han-chat-bitrix-local-app:local
BITRIX_SYNC_IMAGE=han-chat-bitrix-sync:local
KEYCLOAK_IMAGE=han-chat-keycloak:local
SMS_SERVICE_IMAGE=han-chat-sms-service:local
# Managed PostgreSQL is external to Compose. Credential-bearing DSNs are secrets.
HAN_PG_HOST=managed-pg.private.example
HAN_PG_PORT=5433
HAN_PG_DATABASE=han_chat
PG_CA_HOST_PATH=/opt/han-chat/secrets/pg/ca.pem
# PgBouncer session mode uses database-level search_path; runtime DSNs must not pass options/currentSchema.
KEYCLOAK_DB_URL=jdbc:postgresql://managed-pg.private.example:5433/han_chat?sslmode=verify-full&sslrootcert=/run/secrets/pg-ca.pem
KEYCLOAK_DB_SCHEMA=keycloak
KEYCLOAK_DB_USERNAME=keycloak_user
PUBLIC_HOST=chat.example.ru
PUBLIC_WEB_URL=https://chat.example.ru
PUBLIC_API_URL=https://chat.example.ru/api
PUBLIC_AUTH_URL=https://chat.example.ru/auth
NGINX_HTTP_PORT=80
NGINX_HTTPS_PORT=443
NGINX_TLS_ENABLED=true
NGINX_TLS_CERTIFICATE=/etc/letsencrypt/live/chat.example.ru/fullchain.pem
NGINX_TLS_CERTIFICATE_KEY=/etc/letsencrypt/live/chat.example.ru/privkey.pem
NGINX_HSTS_MAX_AGE=31536000
NGINX_CLIENT_MAX_BODY_SIZE=8m
NGINX_RATE_LIMIT_API=60r/m
NGINX_RATE_LIMIT_AUTH=60r/m
NGINX_RATE_LIMIT_PUBLIC=60r/m
NGINX_RATE_LIMIT_POLLING=60r/m
NGINX_RATE_LIMIT_DOWNLOADS=30r/m
NGINX_RATE_LIMIT_NOTIFICATIONS_READ=120r/m
NGINX_RATE_LIMIT_NOTIFICATIONS_ACTION=60r/m
NGINX_RATE_LIMIT_NOTIFICATION_UPLOAD=20r/m
NGINX_RATE_LIMIT_NOTIFICATIONS_PUBLIC=60r/m
NGINX_RATE_LIMIT_BITRIX=120r/m
NGINX_RATE_LIMIT_SMS_CALLBACK=120r/m
NGINX_RATE_LIMIT_WS=30r/m
NGINX_MESSAGE_READ_TIMEOUT_SEC=330
NGINX_TRUSTED_PROXY_CIDR=127.0.0.1/32
TRUSTED_PROXY_CIDRS=172.16.0.0/12
BITRIX_FRAME_ANCESTORS=https://*.bitrix24.ru
S3_CONNECT_SRC=https://*.s3.ru-7.storage.selcloud.ru
FRONTEND_DEV_PROXY_ENABLED=false
# При false значение не используется
EXPO_DEV_SERVER_URL=http://host.docker.internal:8081
ACME_EMAIL=ops@example.ru
KEYCLOAK_PUBLIC_URL=https://chat.example.ru/auth
KEYCLOAK_INTERNAL_URL=http://keycloak:8080/auth
KEYCLOAK_REALM=han-chat
KEYCLOAK_AUDIENCE=han-chat-api
# На мок среде true,true. На продакшн false,false.
KEYCLOAK_OTP_MOCK_ENABLED=true
KEYCLOAK_OTP_MOCK_RISK_ACCEPTED=false
KEYCLOAK_YANDEX_CAPTCHA_ENABLED=false
# Public site key; the server key stays in Secrets Manager when CAPTCHA is enabled.
KEYCLOAK_YANDEX_CAPTCHA_CLIENT_KEY=
KEYCLOAK_OTP_SETTINGS_MAX_STALE_SEC=300
KEYCLOAK_SETTINGS_BRIDGE_URL=http://api-backend:8000/internal/settings/v1/otp
KEYCLOAK_SMS_SERVICE_URL=http://sms-service:8080
KEYCLOAK_ADMIN=bootstrap-admin
REDIS_MAXMEMORY=384mb
REDIS_EVICTION_POLICY=volatile-lru
# i-Digital Direct non-secret configuration.
IDGTL_SMS_BASE_URL=https://direct.i-dgtl.ru
IDGTL_SMS_CALLBACK_PUBLIC_URL=https://chat.example.ru/callbacks/idgtl/sms
BITRIX_LOCAL_APP_BASE_URL=http://bitrix-local-app:8080
BITRIX_API_INBOX_PATH=/internal/openlines/v1/inbox
BITRIX_API_FORWARD_URL=http://api-backend:8000/internal/openlines/v1/inbox
MESSAGE_SAFETY_URL=http://message-safety:8080
MESSAGE_SAFETY_POST_TIMEOUT_SEC=5
MESSAGE_SAFETY_TASK_POLL_INTERVAL_SEC=2
MESSAGE_SAFETY_TASK_POLL_MAX_SEC=300
MESSAGE_SAFETY_TASK_TTL_SEC=900
MESSAGE_SAFETY_FILE_SCAN_TIMEOUT_SEC=60
MESSAGE_SAFETY_RULES_VERSION=2026-01-01
#Отключение синхронизации (при отключенной синхронизации параметры ниже не работают)
BITRIX_SYNC_ENABLED=false
BITRIX_SYNC_CRM_BASE_URL=https://example.bitrix24.ru
BITRIX_SYNC_CONTACT_MAP_INTERVAL_SEC=60
BITRIX_SYNC_CONTACT_UPDATE_INTERVAL_SEC=30
BITRIX_SYNC_CRM_MAX_CONCURRENCY=2
BITRIX_SYNC_CONTACT_LIST_BATCH_SIZE=50
BITRIX_CLIENT_ID=change-me-client-id
BITRIX_CONNECTOR_ID=han_mobile_app
BITRIX_CONNECTOR_NAME=HAN Mobile App
BITRIX_OPEN_LINE_ID=<N>
BITRIX_EXPECTED_DOMAIN=<ваш-портал>.bitrix24.ru
BITRIX_PUBLIC_BASE_URL=https://chat.example.ru/bitrix
BITRIX_HTTP_TIMEOUT_SEC=10
BITRIX_LOCAL_APP_HTTP_TIMEOUT_SEC=20
SELECTEL_S3_ENDPOINT_URL=https://s3.ru-7.storage.selcloud.ru
SELECTEL_S3_BUCKET_DOCUMENTS=han-chat-documents
SELECTEL_S3_BUCKET_ATTACHMENTS=han-chat-attachments
SELECTEL_S3_BUCKET_QUARANTINE=han-chat-quarantine
OTEL_EXPORTER_OTLP_ENDPOINT=http://otel-collector:4317
OTEL_SERVICE_NAME_API=api-backend
OTEL_SERVICE_NAME_SMS_API=sms-service
OTEL_SERVICE_NAME_SMS_WORKER=sms-worker
SMS_METRICS_PORT=9464
#Если есть внешний OTLP-сервис, замените (Точный формат авторизации зависит от провайдера):
#Если внешнего OTLP-сервиса пока нет, otlp.example.invalid:4317 можно временно оставить, но Collector будет постоянно пытаться подключиться, писать предупреждения и накапливать очередь.
OTEL_REMOTE_ENDPOINT=otlp.example.invalid:4317
# true только для plaintext OTLP внутри доверенной приватной сети (например self-hosted SigNoz).
OTEL_REMOTE_TLS_INSECURE=false
# SDK отправляет все spans локальному Collector; решение о хранении принимает tail_sampling.
OTEL_TRACES_SAMPLER=always_on
# OTEL_QUEUE_SIZE пока документирует целевую ёмкость, конфигурация Collector закреплена в YAML.
OTEL_QUEUE_SIZE=10000
VM1_app/codebase/backend/.gitattributes
+7
View File
@@ -0,0 +1,7 @@
*.sh text eol=lf
scripts/validate-env text eol=lf
Dockerfile text eol=lf
*.conf text eol=lf
*.template text eol=lf
*.yml text eol=lf
*.yaml text eol=lf
VM1_app/codebase/backend/.gitignore
+10
View File
@@ -0,0 +1,10 @@
.env
.env.*
!.env.example
*.pem
*.key
*.crt
backups/
__pycache__/
*.pyc
.pytest_cache/
VM1_app/codebase/backend/api-backend/.dockerignore
+8
View File
@@ -0,0 +1,8 @@
.env
.git
.mypy_cache
.pytest_cache
.ruff_cache
__pycache__
*.py[cod]
tests
VM1_app/codebase/backend/api-backend/Dockerfile
+20
View File
@@ -0,0 +1,20 @@
FROM python:3.12-slim AS builder
WORKDIR /build
RUN pip install --no-cache-dir --upgrade pip build
COPY pyproject.toml ./
COPY app ./app
RUN python -m build --wheel
FROM python:3.12-slim
ENV PYTHONDONTWRITEBYTECODE=1 PYTHONUNBUFFERED=1
RUN addgroup --system --gid 10001 han && adduser --system --uid 10001 --ingroup han han
WORKDIR /app
COPY --from=builder /build/dist/*.whl /tmp/
RUN pip install --no-cache-dir /tmp/*.whl && rm -f /tmp/*.whl
COPY alembic.ini ./
COPY alembic ./alembic
COPY --chmod=0555 container-entrypoint.sh /usr/local/bin/han-container-entrypoint
USER 10001:10001
EXPOSE 8000
ENTRYPOINT ["/usr/local/bin/han-container-entrypoint"]
CMD ["uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "8000", "--no-proxy-headers"]
VM1_app/codebase/backend/api-backend/README.md
+84
View File
@@ -0,0 +1,84 @@
# HAN Chat API backend
FastAPI-сервис публичного API, internal Open Lines/settings API, WebSocket realtime,
Message Safety orchestration, S3 lifecycle и PostgreSQL workers.
## Запуск
Python 3.12+:
```bash
python -m venv .venv
.venv/Scripts/pip install -e ".[dev]"
alembic upgrade head
uvicorn app.main:app --host 0.0.0.0 --port 8000
```
Миграции выполняются отдельным deployment step. Приложение не применяет DDL при старте.
Workers запускаются независимо:
```bash
han-delivery-worker
han-safety-worker
han-cleanup-worker
han-notification-expire-worker
han-notification-draft-cleanup-worker
```
## Переменные окружения
Сервис читает только инфраструктурные параметры и секреты из корневого `backend/.env`.
Локальный `.env` не коммитится. Business settings создаются миграцией в
`han_app.app_settings`.
Обязательны:
- `APP_ENV`, `API_PORT`, `LOG_LEVEL`, `DATABASE_URL`;
- `REDIS_URL`, `REDIS_REALTIME_URL`;
- `KEYCLOAK_PUBLIC_URL`, `KEYCLOAK_INTERNAL_URL`, `KEYCLOAK_REALM`,
`KEYCLOAK_AUDIENCE`;
- `MESSAGE_SAFETY_URL`, `MESSAGE_SAFETY_SERVICE_TOKEN`,
`MESSAGE_SAFETY_CA_FILE`, `MESSAGE_SAFETY_API_PREFIX=/internal/safety/v2`,
`MESSAGE_SAFETY_POST_TIMEOUT_SEC`, `MESSAGE_SAFETY_TASK_POLL_INTERVAL_SEC`,
`MESSAGE_SAFETY_TASK_POLL_MAX_SEC`,
`MESSAGE_SAFETY_CIRCUIT_FAILURE_THRESHOLD`, `MESSAGE_SAFETY_CIRCUIT_OPEN_SEC`;
- `BITRIX_LOCAL_APP_BASE_URL`, `BITRIX_LOCAL_APP_INTERNAL_TOKEN`,
`BITRIX_API_INBOX_TOKEN`, `BITRIX_LOCAL_APP_HTTP_TIMEOUT_SEC`,
`BITRIX_LOCAL_APP_CIRCUIT_FAILURE_THRESHOLD`,
`BITRIX_LOCAL_APP_CIRCUIT_OPEN_SEC`;
- `KEYCLOAK_SETTINGS_BRIDGE_TOKEN`;
- `SELECTEL_S3_ENDPOINT_URL`, `SELECTEL_S3_BUCKET_DOCUMENTS`,
`SELECTEL_S3_BUCKET_ATTACHMENTS`, `SELECTEL_S3_BUCKET_QUARANTINE`,
`SELECTEL_S3_ACCESS_KEY`, `SELECTEL_S3_SECRET_KEY`;
- `CURSOR_HMAC_SECRET` — случайный секрет не короче 32 байт;
- `NOTIFICATIONS_TOKEN_PRODUCER_TEST` — отдельный bearer token тестового
продюсера Notification Center; в БД синхронизируется только SHA-256 hash;
- `OTEL_EXPORTER_OTLP_ENDPOINT` — опциональный endpoint collector.
Токены генерируются `openssl rand -hex 32`. S3 read-only credentials Message Safety
не передаются этому контейнеру. В production подключение PostgreSQL должно использовать
TLS. Target `MESSAGE_SAFETY_URL=https://processing.internal:8443`; certificate
проверяется по internal CA, plaintext HTTP запрещён. Текущий Docker hostname
`message-safety` относится только к legacy stub до cutover.
Smoke-сценарий `producer_test`: отправить `POST
/internal/notifications/v1/notifications` с `Authorization: Bearer
$NOTIFICATIONS_TOKEN_PRODUCER_TEST`, `source=producer_test` и уникальным
`external_id`; повтор того же тела вернёт `200`. Затем передать ту же пару
`source`/`external_id` в `POST /internal/notifications/v1/notifications/cancel`
с `close_reason=cancelled`; повторная отмена также вернёт `200`.
## Проверки
```bash
ruff check .
ruff format --check .
mypy app
pytest
```
`/health/live` проверяет процесс. `/health/ready` проверяет критические
зависимости read API. Remote Message Safety не выключает чтение/общую readiness:
send endpoint отдельно проверяет требуемую capability и fail-closed возвращает
`503`, если ВМ2 недоступна.
VM1_app/codebase/backend/api-backend/alembic.ini
+37
View File
@@ -0,0 +1,37 @@
[alembic]
script_location = alembic
prepend_sys_path = .
path_separator = os
[loggers]
keys = root,sqlalchemy,alembic
[handlers]
keys = console
[formatters]
keys = generic
[logger_root]
level = WARN
handlers = console
[logger_sqlalchemy]
level = WARN
handlers =
qualname = sqlalchemy.engine
[logger_alembic]
level = INFO
handlers =
qualname = alembic
[handler_console]
class = StreamHandler
args = (sys.stderr,)
level = NOTSET
formatter = generic
[formatter_generic]
format = %(levelname)-5.5s [%(name)s] %(message)s
datefmt = %H:%M:%S
VM1_app/codebase/backend/api-backend/alembic/env.py
+49
View File
@@ -0,0 +1,49 @@
import asyncio
import os
from logging.config import fileConfig
from sqlalchemy import pool
from alembic import context
from app.db import Base
from app.postgres import create_postgres_engine
config = context.config
if config.config_file_name:
fileConfig(config.config_file_name)
database_url = os.environ["DATABASE_URL"]
config.set_main_option("sqlalchemy.url", database_url.replace("%", "%%"))
target_metadata = Base.metadata
def do_run_migrations(connection) -> None:
context.configure(
connection=connection,
target_metadata=target_metadata,
include_schemas=True,
version_table_schema="han_app",
compare_type=True,
)
with context.begin_transaction():
context.run_migrations()
async def run_async_migrations() -> None:
connectable = create_postgres_engine(database_url, poolclass=pool.NullPool)
async with connectable.connect() as connection:
await connection.run_sync(do_run_migrations)
await connectable.dispose()
if context.is_offline_mode():
context.configure(
url=config.get_main_option("sqlalchemy.url"),
target_metadata=target_metadata,
literal_binds=True,
include_schemas=True,
version_table_schema="han_app",
)
with context.begin_transaction():
context.run_migrations()
else:
asyncio.run(run_async_migrations())
VM1_app/codebase/backend/api-backend/alembic/versions/0001_initial_han_app.py
+182
View File
@@ -0,0 +1,182 @@
"""Initial han_app schema, seed and CRM sync triggers.
Revision ID: 0001_initial
Revises:
Create Date: 2026-07-10
"""
from collections.abc import Sequence
import sqlalchemy as sa
from alembic import op
from app.db import Base
revision: str = "0001_initial"
down_revision: str | None = None
branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None
SEED = {
"auth.phone.enabled": ("true", "boolean", True),
"auth.password.enabled": ("false", "boolean", True),
"otp.phone.max_send_attempts_per_24h": ("3", "integer", False),
"otp.phone.min_seconds_between_attempts": ("30", "integer", False),
"operator.call.phone": ("+74999591007", "string", True),
"consent.personal_data.required": ("true", "boolean", True),
"consent.personal_data.document_url": (
"https://www.han0107.ru/privacy/persdata-agree-mobile",
"string",
True,
),
"consent.personal_data.version": ("2026-06-10", "string", True),
"consent.privacy_policy.document_url": (
"https://www.han0107.ru/privacy",
"string",
True,
),
"consent.user_agreement.required": ("true", "boolean", True),
"consent.user_agreement.document_url": (
"https://www.han0107.ru/user-agreement",
"string",
True,
),
"consent.user_agreement.version": ("2026-06-10", "string", True),
"consent.marketing.required": ("false", "boolean", True),
"consent.marketing.document_url": (
"https://www.han0107.ru/privacy/ads-agree",
"string",
True,
),
"consent.marketing.version": ("2026-06-10", "string", True),
"chat.attachments.allowed_extensions": (
"jpg,jpeg,png,webp,heic,heif,pdf",
"string_list",
True,
),
"chat.attachments.allowed_mime_types": (
"image/jpeg,image/png,image/webp,image/heic,image/heif,application/pdf",
"string_list",
True,
),
"chat.attachments.disallowed_extensions": ("svg,doc,docx,xls,xlsx,csv", "string_list", False),
"chat.attachments.max_size_mb": ("5", "integer", True),
"chat.attachments.storage": ("selectel_s3", "string", False),
"chat.attachments.upload_mode": ("presigned_put", "string", False),
"chat.attachments.safety_scan_required": ("true", "boolean", False),
"chat.attachments.presigned_upload_ttl_seconds": ("600", "integer", False),
"rate_limit.message_send.per_user": ("30/minute", "string", False),
"rate_limit.message_send.per_dialog": ("20/minute", "string", False),
"rate_limit.download_url.per_user": ("60/hour", "string", False),
"rate_limit.public_endpoints.per_ip": ("60/minute", "string", False),
"rate_limit.login.per_ip": ("10/minute", "string", False),
"ux.session.idle_timeout_minutes": ("30", "integer", True),
"security.cors.allowed_origins": ("https://tohin.ru", "string_list", False),
"security.public_cache.max_age_seconds": ("3600", "integer", False),
}
def upgrade() -> None:
bind = op.get_bind()
op.execute("CREATE EXTENSION IF NOT EXISTS pgcrypto")
op.execute("CREATE SCHEMA IF NOT EXISTS han_app")
Base.metadata.create_all(bind=bind, checkfirst=True)
op.execute(
"""
CREATE UNIQUE INDEX IF NOT EXISTS uq_dialog_one_active_per_user
ON han_app.dialogs(user_id)
WHERE record_status='A'
AND status IN ('open','waiting_for_company','waiting_for_client')
"""
)
op.execute(
"""
CREATE UNIQUE INDEX IF NOT EXISTS uq_profiles_active_bitrix_contact
ON han_app.client_profiles(bitrix_contact_id)
WHERE bitrix_contact_id IS NOT NULL AND record_status='A'
"""
)
op.execute(
"""
CREATE OR REPLACE FUNCTION han_app.enqueue_contact_sync()
RETURNS trigger
LANGUAGE plpgsql
SECURITY INVOKER
SET search_path = han_app, pg_temp
AS $$
DECLARE
v_entity_id uuid;
v_task_type text;
v_dedup text;
BEGIN
IF current_setting('han.sync_suppress', true) = 'true' THEN
RETURN NEW;
END IF;
IF TG_TABLE_NAME = 'user_identities' THEN
v_entity_id := NEW.id;
v_task_type := CASE WHEN TG_OP = 'INSERT'
THEN 'contact.map_or_create' ELSE 'contact.update' END;
ELSE
v_entity_id := NEW.user_id;
v_task_type := CASE WHEN TG_OP = 'INSERT'
THEN 'contact.map_or_create' ELSE 'contact.update' END;
END IF;
v_dedup := v_task_type || ':' || v_entity_id::text || ':' ||
encode(public.digest(row_to_json(NEW)::text, 'sha256'), 'hex');
INSERT INTO han_app.sync_queue
(id, task_type, entity_type, entity_id, dedup_key, payload_json,
status, attempt_count, next_attempt_at, created_at, updated_at)
VALUES
(gen_random_uuid(), v_task_type, 'contact', v_entity_id, v_dedup,
jsonb_build_object('entity_id', v_entity_id), 'pending', 0,
now(), now(), now())
ON CONFLICT (dedup_key) DO NOTHING;
RETURN NEW;
END;
$$
"""
)
op.execute(
"DROP TRIGGER IF EXISTS trg_identity_contact_sync ON han_app.user_identities"
)
op.execute(
"""
CREATE TRIGGER trg_identity_contact_sync
AFTER INSERT OR UPDATE OF phone_number, record_status
ON han_app.user_identities
FOR EACH ROW EXECUTE FUNCTION han_app.enqueue_contact_sync()
"""
)
op.execute(
"DROP TRIGGER IF EXISTS trg_profile_contact_sync ON han_app.client_profiles"
)
op.execute(
"""
CREATE TRIGGER trg_profile_contact_sync
AFTER INSERT OR UPDATE OF full_name, citizenship, russian_phone,
foreign_phone, email, record_status
ON han_app.client_profiles
FOR EACH ROW EXECUTE FUNCTION han_app.enqueue_contact_sync()
"""
)
for key, (value, value_type, public) in SEED.items():
bind.execute(
sa.text(
"""
INSERT INTO han_app.app_settings
(setting_key, setting_value, value_type, is_public, record_status, updated_at)
VALUES (:key, :value, :value_type, :public, 'A', now())
ON CONFLICT (setting_key) DO UPDATE SET
setting_value = EXCLUDED.setting_value,
value_type = EXCLUDED.value_type,
is_public = EXCLUDED.is_public,
record_status = 'A',
updated_at = now()
"""
),
{"key": key, "value": value, "value_type": value_type, "public": public},
)
def downgrade() -> None:
raise RuntimeError("Initial data migration is forward-only")
VM1_app/codebase/backend/api-backend/alembic/versions/0002_qualify_pgcrypto_digest.py
+62
View File
@@ -0,0 +1,62 @@
"""Qualify pgcrypto digest in the contact sync trigger.
Revision ID: 0002_pgcrypto_digest
Revises: 0001_initial
Create Date: 2026-07-16
"""
from collections.abc import Sequence
from alembic import op
revision: str = "0002_pgcrypto_digest"
down_revision: str | None = "0001_initial"
branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None
def upgrade() -> None:
op.execute(
"""
CREATE OR REPLACE FUNCTION han_app.enqueue_contact_sync()
RETURNS trigger
LANGUAGE plpgsql
SECURITY INVOKER
SET search_path = han_app, pg_temp
AS $$
DECLARE
v_entity_id uuid;
v_task_type text;
v_dedup text;
BEGIN
IF current_setting('han.sync_suppress', true) = 'true' THEN
RETURN NEW;
END IF;
IF TG_TABLE_NAME = 'user_identities' THEN
v_entity_id := NEW.id;
v_task_type := CASE WHEN TG_OP = 'INSERT'
THEN 'contact.map_or_create' ELSE 'contact.update' END;
ELSE
v_entity_id := NEW.user_id;
v_task_type := CASE WHEN TG_OP = 'INSERT'
THEN 'contact.map_or_create' ELSE 'contact.update' END;
END IF;
v_dedup := v_task_type || ':' || v_entity_id::text || ':' ||
encode(public.digest(row_to_json(NEW)::text, 'sha256'), 'hex');
INSERT INTO han_app.sync_queue
(id, task_type, entity_type, entity_id, dedup_key, payload_json,
status, attempt_count, next_attempt_at, created_at, updated_at)
VALUES
(gen_random_uuid(), v_task_type, 'contact', v_entity_id, v_dedup,
jsonb_build_object('entity_id', v_entity_id), 'pending', 0,
now(), now(), now())
ON CONFLICT (dedup_key) DO NOTHING;
RETURN NEW;
END;
$$
"""
)
def downgrade() -> None:
raise RuntimeError("Contact sync trigger migration is forward-only")
VM1_app/codebase/backend/api-backend/alembic/versions/0003_consent_device_audit_context.py
+33
View File
@@ -0,0 +1,33 @@
"""Store consent device snapshots and remove audit IP.
Revision ID: 0003_consent_audit
Revises: 0002_pgcrypto_digest
Create Date: 2026-07-16
"""
from collections.abc import Sequence
from alembic import op
revision: str = "0003_consent_audit"
down_revision: str | None = "0002_pgcrypto_digest"
branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None
def upgrade() -> None:
op.execute(
"""
ALTER TABLE han_app.user_consents
ADD COLUMN IF NOT EXISTS device_json jsonb NOT NULL DEFAULT '{}'::jsonb
"""
)
op.execute(
"""
ALTER TABLE han_app.audit_events
DROP COLUMN IF EXISTS ip
"""
)
def downgrade() -> None:
raise RuntimeError("Consent and audit context migration is forward-only")
VM1_app/codebase/backend/api-backend/alembic/versions/0004_device_and_otp_limits.py
+40
View File
@@ -0,0 +1,40 @@
"""Store raw device identifiers and add OTP verification limit.
Revision ID: 0004_device_otp
Revises: 0003_consent_audit
Create Date: 2026-07-21
"""
from collections.abc import Sequence
from alembic import op
revision: str = "0004_device_otp"
down_revision: str | None = "0003_consent_audit"
branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None
def upgrade() -> None:
op.execute(
"""
ALTER TABLE han_app.ux_sessions
ADD COLUMN IF NOT EXISTS device_id varchar(255)
"""
)
op.execute(
"""
INSERT INTO han_app.app_settings
(setting_key, setting_value, value_type, is_public, description,
record_status, updated_at)
VALUES
('otp.phone.max_verify_attempts', '5', 'integer', false,
'Maximum failed verification attempts for one OTP challenge',
'A', now())
ON CONFLICT (setting_key) DO NOTHING
"""
)
def downgrade() -> None:
raise RuntimeError("Device and OTP limits migration is forward-only")
VM1_app/codebase/backend/api-backend/alembic/versions/0005_otp_runtime_settings.py
+37
View File
@@ -0,0 +1,37 @@
"""Seed runtime OTP settings.
Revision ID: 0005_otp_settings
Revises: 0004_device_otp
Create Date: 2026-07-22
"""
from collections.abc import Sequence
from alembic import op
revision: str = "0005_otp_settings"
down_revision: str | None = "0004_device_otp"
branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None
def upgrade() -> None:
op.execute(
"""
INSERT INTO han_app.app_settings
(setting_key, setting_value, value_type, is_public, description,
record_status, updated_at)
VALUES
('otp.phone.code_length', '6', 'integer', false,
'Length of the numeric phone OTP', 'A', now()),
('otp.phone.ttl_seconds', '60', 'integer', false,
'Phone OTP lifetime from durable order time', 'A', now()),
('otp.phone.sms_order_timeout_ms', '3000', 'integer', false,
'Keycloak timeout for durable SMS order creation', 'A', now())
ON CONFLICT (setting_key) DO NOTHING
"""
)
def downgrade() -> None:
raise RuntimeError("OTP runtime settings migration is forward-only")
VM1_app/codebase/backend/api-backend/alembic/versions/0006_consent_privacy_policy_url.py
+34
View File
@@ -0,0 +1,34 @@
"""Seed consent.privacy_policy.document_url for personal_data consent UI.
Revision ID: 0006_privacy_policy
Revises: 0005_otp_settings
Create Date: 2026-07-23
"""
from collections.abc import Sequence
from alembic import op
revision: str = "0006_privacy_policy"
down_revision: str | None = "0005_otp_settings"
branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None
def upgrade() -> None:
op.execute(
"""
INSERT INTO han_app.app_settings
(setting_key, setting_value, value_type, is_public, description,
record_status, updated_at)
VALUES
('consent.privacy_policy.document_url',
'https://www.han0107.ru/privacy', 'string', true,
'Privacy policy URL shown next to personal_data consent', 'A', now())
ON CONFLICT (setting_key) DO NOTHING
"""
)
def downgrade() -> None:
raise RuntimeError("Privacy policy consent URL migration is forward-only")
VM1_app/codebase/backend/api-backend/alembic/versions/0007_consent_marketing_document_url.py
+34
View File
@@ -0,0 +1,34 @@
"""Seed consent.marketing.document_url for marketing consent UI link.
Revision ID: 0007_marketing_doc
Revises: 0006_privacy_policy
Create Date: 2026-07-23
"""
from collections.abc import Sequence
from alembic import op
revision: str = "0007_marketing_doc"
down_revision: str | None = "0006_privacy_policy"
branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None
def upgrade() -> None:
op.execute(
"""
INSERT INTO han_app.app_settings
(setting_key, setting_value, value_type, is_public, description,
record_status, updated_at)
VALUES
('consent.marketing.document_url',
'https://www.han0107.ru/privacy/ads-agree', 'string', true,
'Marketing communications consent document URL', 'A', now())
ON CONFLICT (setting_key) DO NOTHING
"""
)
def downgrade() -> None:
raise RuntimeError("Marketing consent document URL migration is forward-only")
VM1_app/codebase/backend/api-backend/alembic/versions/0008_notification_center_v1.py
+651
View File
@@ -0,0 +1,651 @@
"""Notification Center v1 schema, catalog and settings.
Revision ID: 0008_notifications_v1
Revises: 0007_marketing_doc
Create Date: 2026-07-27
"""
import hashlib
import os
from collections.abc import Sequence
import sqlalchemy as sa
from alembic import op
revision: str = "0008_notifications_v1"
down_revision: str | None = "0007_marketing_doc"
branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None
COMMON = """
record_status varchar(1) NOT NULL DEFAULT 'A',
status_changed_at timestamptz NULL,
status_change_reason varchar(255) NULL,
created_at timestamptz NOT NULL DEFAULT now(),
updated_at timestamptz NOT NULL DEFAULT now(),
updater_user_id uuid NULL
"""
def _execute_batch(sql: str) -> None:
"""Execute one statement at a time for the asyncpg prepared-statement dialect."""
statement: list[str] = []
in_dollar_quote = False
index = 0
while index < len(sql):
if sql[index : index + 2] == "$$":
in_dollar_quote = not in_dollar_quote
statement.append("$$")
index += 2
continue
character = sql[index]
if character == ";" and not in_dollar_quote:
value = "".join(statement).strip()
if value:
op.execute(value)
statement.clear()
else:
statement.append(character)
index += 1
value = "".join(statement).strip()
if value:
op.execute(value)
def upgrade() -> None:
bind = op.get_bind()
_execute_batch(
f"""
CREATE TABLE han_app.notification_cta_actions (
id uuid PRIMARY KEY, code varchar(32) NOT NULL UNIQUE,
description varchar(255) NOT NULL, requires_auth boolean NOT NULL,
required_instance_fields varchar(64)[] NOT NULL DEFAULT '{{}}', {COMMON}
);
CREATE TABLE han_app.notification_buttons (
id uuid PRIMARY KEY, code varchar(32) NOT NULL UNIQUE,
label varchar(64) NOT NULL, sets_hidden boolean NOT NULL DEFAULT false,
applies_hidden_ttl boolean NOT NULL DEFAULT false,
close_reason varchar(32), submits_documents boolean NOT NULL DEFAULT false,
{COMMON},
CHECK (NOT applies_hidden_ttl OR sets_hidden),
CHECK (NOT submits_documents OR close_reason IS NOT NULL)
);
CREATE TABLE han_app.notification_color_tokens (
id uuid PRIMARY KEY, code varchar(32) NOT NULL UNIQUE,
description varchar(255) NOT NULL, sort_order smallint NOT NULL, {COMMON}
);
CREATE TABLE han_app.notification_types (
id uuid PRIMARY KEY, code varchar(32) NOT NULL UNIQUE,
contour varchar(1) NOT NULL CHECK (contour IN ('G','P')),
priority smallint NOT NULL, countable boolean NOT NULL,
label varchar(64) NOT NULL,
color_token varchar(32) NOT NULL REFERENCES han_app.notification_color_tokens(code)
ON DELETE RESTRICT,
icon_code varchar(32), cta_text varchar(64) NOT NULL,
cta_action varchar(32) NOT NULL REFERENCES han_app.notification_cta_actions(code)
ON DELETE RESTRICT,
cta_sets_hidden boolean NOT NULL DEFAULT false,
cta_close_reason varchar(32),
button_primary_code varchar(32) REFERENCES han_app.notification_buttons(code)
ON DELETE RESTRICT,
button_secondary_code varchar(32) REFERENCES han_app.notification_buttons(code)
ON DELETE RESTRICT,
hidden_ttl_days smallint,
documents_allowed boolean NOT NULL DEFAULT false,
hide_on_document_download boolean NOT NULL DEFAULT false,
required_detail_blocks varchar(64)[] NOT NULL DEFAULT '{{}}',
{COMMON},
CHECK (contour <> 'G' OR countable = false),
CHECK (contour <> 'G' OR cta_action <> 'open_detail'),
CHECK (cta_action <> 'open_detail' OR
(cta_sets_hidden = false AND cta_close_reason IS NULL)),
CHECK (cta_action = 'open_detail' OR
(documents_allowed = false AND hide_on_document_download = false
AND required_detail_blocks = '{{}}')),
CHECK (NOT hide_on_document_download OR documents_allowed),
CHECK (cta_action <> 'open_detail' OR button_primary_code IS NOT NULL),
CHECK (cta_action = 'open_detail' OR
(button_primary_code IS NULL AND button_secondary_code IS NULL)),
CHECK (button_secondary_code IS NULL OR button_primary_code IS NOT NULL),
CHECK (button_secondary_code IS NULL OR
button_secondary_code <> button_primary_code)
);
CREATE TABLE han_app.notification_sources (
id uuid PRIMARY KEY, code varchar(32) NOT NULL UNIQUE,
description text, token_hash varchar(128) NOT NULL UNIQUE,
token_rotated_at timestamptz, {COMMON}
);
CREATE TABLE han_app.notifications (
id uuid PRIMARY KEY,
user_id uuid NOT NULL REFERENCES han_app.user_identities(id) ON DELETE RESTRICT,
notification_type varchar(32) NOT NULL
REFERENCES han_app.notification_types(code) ON DELETE RESTRICT,
source varchar(32) NOT NULL
REFERENCES han_app.notification_sources(code) ON DELETE RESTRICT,
external_id varchar(128) NOT NULL,
request_fingerprint varchar(64) NOT NULL,
notification_datetime timestamptz NOT NULL,
header varchar(255) NOT NULL, text varchar(1024),
priority_override smallint, date_expired timestamptz,
price numeric(12,2), old_price numeric(12,2), payment_url text,
details jsonb, details_schema_version smallint NOT NULL DEFAULT 1,
chat_message_text varchar(1024),
lifecycle_status varchar(16) NOT NULL DEFAULT 'active'
CHECK (lifecycle_status IN ('active','closed')),
visibility varchar(16) NOT NULL DEFAULT 'visible'
CHECK (visibility IN ('visible','hidden')),
is_read boolean NOT NULL DEFAULT false,
close_reason varchar(32) CHECK (close_reason IS NULL OR close_reason IN
('user_done','docs_submitted','offer_accepted','paid','expired','cancelled')),
closed_at timestamptz, {COMMON},
CONSTRAINT uq_notifications_source_key UNIQUE (source, external_id),
CHECK (old_price IS NULL OR price IS NOT NULL),
CHECK (lifecycle_status <> 'closed' OR
(close_reason IS NOT NULL AND closed_at IS NOT NULL))
);
CREATE INDEX ix_notifications_user_active
ON han_app.notifications
(user_id, lifecycle_status, visibility, notification_datetime DESC, id DESC)
WHERE record_status='A';
CREATE INDEX ix_notifications_expire ON han_app.notifications(date_expired)
WHERE record_status='A' AND lifecycle_status='active'
AND date_expired IS NOT NULL;
CREATE TABLE han_app.guest_notifications (
id uuid PRIMARY KEY,
notification_type varchar(32) NOT NULL
REFERENCES han_app.notification_types(code) ON DELETE RESTRICT,
notification_datetime timestamptz NOT NULL,
header varchar(255) NOT NULL, text varchar(1024),
priority_override smallint, date_expired timestamptz,
price numeric(12,2), old_price numeric(12,2),
instruction_url text, chat_message_text varchar(1024),
lifecycle_status varchar(16) NOT NULL DEFAULT 'active'
CHECK (lifecycle_status IN ('active','closed')),
closed_at timestamptz, {COMMON},
CHECK (old_price IS NULL OR price IS NOT NULL),
CHECK (instruction_url IS NULL OR instruction_url LIKE 'https://%')
);
CREATE INDEX ix_guest_notifications_active
ON han_app.guest_notifications(lifecycle_status, notification_datetime DESC, id DESC)
WHERE record_status='A';
CREATE INDEX ix_guest_notifications_expire
ON han_app.guest_notifications(date_expired)
WHERE record_status='A' AND lifecycle_status='active'
AND date_expired IS NOT NULL;
CREATE TABLE han_app.notification_documents (
id uuid PRIMARY KEY,
notification_id uuid NOT NULL
REFERENCES han_app.notifications(id) ON DELETE RESTRICT,
document_id uuid NOT NULL REFERENCES han_app.documents(id) ON DELETE RESTRICT,
sort_order smallint NOT NULL DEFAULT 0,
download_url_issued_at timestamptz, {COMMON},
UNIQUE (notification_id, document_id)
);
CREATE TABLE han_app.client_upload_drafts (
id uuid PRIMARY KEY,
user_id uuid NOT NULL REFERENCES han_app.user_identities(id) ON DELETE RESTRICT,
context_type varchar(32) NOT NULL CHECK (context_type IN ('notification')),
context_id uuid NOT NULL,
original_file_name varchar(255) NOT NULL,
safe_file_name varchar(255) NOT NULL,
mime_type varchar(128) NOT NULL,
size_bytes bigint NOT NULL CHECK (size_bytes > 0),
checksum_sha256 char(64),
scan_status varchar(16) NOT NULL DEFAULT 'pending'
CHECK (scan_status IN ('pending','clean','infected','failed')),
storage_bucket varchar(255) NOT NULL,
object_key varchar(1024) NOT NULL,
quarantine_object_key varchar(1024),
upload_expires_at timestamptz, completed_at timestamptz,
state varchar(16) NOT NULL DEFAULT 'draft'
CHECK (state IN ('draft','submitted','discarded')),
submission_id uuid,
created_at timestamptz NOT NULL DEFAULT now(),
updated_at timestamptz NOT NULL DEFAULT now()
);
CREATE INDEX ix_client_upload_drafts_context
ON han_app.client_upload_drafts(user_id, context_type, context_id)
WHERE state='draft';
CREATE INDEX ix_client_upload_drafts_scan
ON han_app.client_upload_drafts(scan_status, updated_at);
CREATE INDEX ix_client_upload_drafts_created
ON han_app.client_upload_drafts(created_at);
CREATE TABLE han_app.client_documents (
id uuid PRIMARY KEY,
user_id uuid NOT NULL REFERENCES han_app.user_identities(id) ON DELETE RESTRICT,
context_type varchar(32) NOT NULL, context_id uuid NOT NULL,
submission_id uuid NOT NULL, source_draft_id uuid NOT NULL UNIQUE,
original_file_name varchar(255) NOT NULL, safe_file_name varchar(255) NOT NULL,
mime_type varchar(128) NOT NULL, size_bytes bigint NOT NULL,
checksum_sha256 char(64) NOT NULL, storage_bucket varchar(255) NOT NULL,
object_key varchar(1024) NOT NULL, submitted_at timestamptz NOT NULL,
{COMMON}, UNIQUE (storage_bucket, object_key)
);
CREATE INDEX ix_client_documents_context
ON han_app.client_documents(context_type, context_id);
CREATE INDEX ix_client_documents_user_submitted
ON han_app.client_documents(user_id, submitted_at DESC);
"""
)
_execute_batch(
"""
CREATE OR REPLACE FUNCTION han_app.validate_guest_notification()
RETURNS trigger LANGUAGE plpgsql SECURITY INVOKER
SET search_path = han_app, pg_temp AS $$
DECLARE v_contour varchar(1); v_action varchar(32); v_required varchar(64)[];
BEGIN
SELECT nt.contour, nt.cta_action, ca.required_instance_fields
INTO v_contour, v_action, v_required
FROM han_app.notification_types nt
JOIN han_app.notification_cta_actions ca ON ca.code=nt.cta_action
WHERE nt.code=NEW.notification_type AND nt.record_status='A';
IF v_contour IS DISTINCT FROM 'G' THEN
RAISE EXCEPTION 'notification type must use guest contour';
END IF;
IF ('instruction_url'=ANY(v_required)) <> (NEW.instruction_url IS NOT NULL)
OR ('chat_message_text'=ANY(v_required)) <>
(NEW.chat_message_text IS NOT NULL) THEN
RAISE EXCEPTION 'guest notification fields do not match CTA';
END IF;
RETURN NEW;
END $$;
CREATE TRIGGER trg_validate_guest_notification
BEFORE INSERT OR UPDATE ON han_app.guest_notifications
FOR EACH ROW EXECUTE FUNCTION han_app.validate_guest_notification();
CREATE OR REPLACE FUNCTION han_app.enqueue_client_document_sync()
RETURNS trigger LANGUAGE plpgsql SECURITY INVOKER
SET search_path = han_app, pg_temp AS $$
BEGIN
IF current_setting('han.sync_suppress', true) = 'true' THEN RETURN NEW; END IF;
INSERT INTO han_app.sync_queue
(id, task_type, entity_type, entity_id, dedup_key, payload_json,
status, attempt_count, next_attempt_at, created_at, updated_at)
VALUES (
gen_random_uuid(), 'document.client_uploaded', 'client_document', NEW.id,
'document.client_uploaded:' || NEW.id::text,
jsonb_build_object(
'client_document_id', NEW.id, 'user_id', NEW.user_id,
'context_type', NEW.context_type, 'context_id', NEW.context_id,
'submission_id', NEW.submission_id, 'storage_bucket', NEW.storage_bucket,
'object_key', NEW.object_key, 'original_file_name', NEW.original_file_name,
'mime_type', NEW.mime_type, 'size_bytes', NEW.size_bytes,
'checksum_sha256', NEW.checksum_sha256),
'pending', 0, now(), now(), now())
ON CONFLICT (dedup_key) DO NOTHING;
RETURN NEW;
END $$;
CREATE TRIGGER trg_client_document_sync
AFTER INSERT ON han_app.client_documents
FOR EACH ROW WHEN (NEW.record_status='A')
EXECUTE FUNCTION han_app.enqueue_client_document_sync();
"""
)
_seed_catalog(bind)
_seed_settings(bind)
def _seed_catalog(bind: sa.Connection) -> None:
# asyncpg rejects multiple SQL commands in one prepared statement.
_execute_batch(
"""
INSERT INTO han_app.notification_cta_actions
(id,code,description,requires_auth,required_instance_fields)
VALUES
(gen_random_uuid(),'open_detail','Open notification detail',true,ARRAY['details']),
(gen_random_uuid(),'open_payment_url','Open payment URL',true,ARRAY['payment_url']),
(gen_random_uuid(),'send_chat_message','Send prepared chat message',true,
ARRAY['chat_message_text']),
(gen_random_uuid(),'start_auth','Start authentication',false,ARRAY[]::varchar[]),
(gen_random_uuid(),'install_app_prompt','Install application or open instruction',
false,ARRAY['instruction_url']);
INSERT INTO han_app.notification_buttons
(id,code,label,sets_hidden,applies_hidden_ttl,close_reason,submits_documents)
VALUES
(gen_random_uuid(),'done','Готово',false,false,'user_done',false),
(gen_random_uuid(),'later','Сделаю позже',false,false,NULL,false),
(gen_random_uuid(),'gotit','Понятно',true,true,NULL,false),
(gen_random_uuid(),'send_docs','Отправить документы',false,false,
'docs_submitted',true);
INSERT INTO han_app.notification_color_tokens
(id,code,description,sort_order)
VALUES
(gen_random_uuid(),'critical','Requires immediate attention',1),
(gen_random_uuid(),'warning','Waiting for a client action',2),
(gen_random_uuid(),'success','Successful result',3),
(gen_random_uuid(),'info','Informational notification',4),
(gen_random_uuid(),'promo','Marketing offer',5),
(gen_random_uuid(),'neutral','Neutral interface hint',6);
"""
)
types = [
(
"authorize",
"G",
1,
False,
"Гостевой режим",
"neutral",
"login",
"Войти →",
"start_auth",
False,
None,
None,
None,
False,
False,
[],
),
(
"install_app",
"G",
2,
False,
"Приложение",
"neutral",
"install",
"Установить →",
"install_app_prompt",
False,
None,
None,
None,
False,
False,
[],
),
(
"promo_global",
"G",
3,
False,
"Акция",
"promo",
"promo",
"Узнать подробнее →",
"send_chat_message",
False,
None,
None,
None,
False,
False,
[],
),
(
"ads_global",
"G",
4,
False,
"Предложение",
"promo",
"offer",
"Узнать подробнее →",
"send_chat_message",
False,
None,
None,
None,
False,
False,
[],
),
(
"urgent",
"P",
1,
True,
"Срочно",
"critical",
"urgent",
"Подробнее →",
"open_detail",
False,
None,
"done",
"later",
False,
False,
[],
),
(
"payment_pending",
"P",
2,
True,
"Оплата",
"warning",
"payment",
"Оплатить →",
"open_payment_url",
False,
None,
None,
None,
False,
False,
[],
),
(
"docs_required",
"P",
2,
True,
"Требуются документы",
"warning",
"upload",
"Загрузить документы →",
"open_detail",
False,
None,
"send_docs",
"later",
False,
False,
[],
),
(
"docs_ready",
"P",
3,
True,
"Документы готовы",
"success",
"download",
"Скачать →",
"open_detail",
False,
None,
"gotit",
None,
True,
True,
["documents"],
),
(
"status_changed",
"P",
3,
True,
"Статус",
"info",
"status",
"Подробнее →",
"open_detail",
False,
None,
"gotit",
None,
False,
False,
[],
),
(
"reminder",
"P",
3,
True,
"Напоминание",
"info",
"reminder",
"Подробнее →",
"open_detail",
False,
None,
"done",
"later",
False,
False,
[],
),
(
"news",
"P",
4,
True,
"Новость",
"info",
"news",
"Подробнее →",
"open_detail",
False,
None,
"gotit",
None,
False,
False,
[],
),
(
"promo_personal",
"P",
5,
True,
"Акция",
"promo",
"promo",
"Узнать подробнее →",
"send_chat_message",
True,
"offer_accepted",
None,
None,
False,
False,
[],
),
(
"ads_personal",
"P",
5,
True,
"Предложение",
"promo",
"offer",
"Узнать подробнее →",
"send_chat_message",
True,
"offer_accepted",
None,
None,
False,
False,
[],
),
]
statement = sa.text(
"""
INSERT INTO han_app.notification_types
(id,code,contour,priority,countable,label,color_token,icon_code,cta_text,cta_action,
cta_sets_hidden,cta_close_reason,button_primary_code,button_secondary_code,
documents_allowed,hide_on_document_download,required_detail_blocks)
VALUES
(gen_random_uuid(),:code,:contour,:priority,:countable,:label,:color,:icon,:cta_text,
:action,:sets_hidden,:close_reason,:primary,:secondary,:documents_allowed,
:hide_download,:required)
"""
)
for row in types:
bind.execute(
statement,
dict(
zip(
(
"code",
"contour",
"priority",
"countable",
"label",
"color",
"icon",
"cta_text",
"action",
"sets_hidden",
"close_reason",
"primary",
"secondary",
"documents_allowed",
"hide_download",
"required",
),
row,
strict=True,
)
),
)
token = os.getenv("NOTIFICATIONS_TOKEN_PRODUCER_TEST", "")
token_hash = (
hashlib.sha256(token.encode()).hexdigest()
if token
else hashlib.sha256(b"disabled:producer_test").hexdigest()
)
bind.execute(
sa.text(
"""
INSERT INTO han_app.notification_sources
(id,code,description,token_hash,token_rotated_at)
VALUES (gen_random_uuid(),'producer_test','Notification Center smoke producer',
:token_hash,now())
"""
),
{"token_hash": token_hash},
)
def _seed_settings(bind: sa.Connection) -> None:
settings = [
("notification.home.max_items", "7", "integer", False),
("notification.center.max_items", "15", "integer", False),
("notification.carousel.autoplay_enabled", "false", "boolean", True),
("notification.carousel.autoplay_interval_ms", "5000", "integer", True),
("notification.hidden.default_ttl_days", "3", "integer", False),
("notification.documents.max_files", "10", "integer", False),
("notification.instruction.allowed_hosts", "chat.example.ru", "string_list", False),
("notification.expire_job.run_at", "00:01", "string", False),
("notification.upload_draft.ttl_days", "7", "integer", False),
("rate_limit.notifications_read.per_user", "120/minute", "string", False),
("rate_limit.notifications_action.per_user", "60/minute", "string", False),
("rate_limit.notification_upload.per_user", "20/minute", "string", False),
("rate_limit.notifications_public.per_ip", "60/minute", "string", False),
]
statement = sa.text(
"""
INSERT INTO han_app.app_settings
(setting_key,setting_value,value_type,is_public,record_status,updated_at)
VALUES (:key,:value,:kind,:public,'A',now())
ON CONFLICT (setting_key) DO UPDATE SET setting_value=EXCLUDED.setting_value,
value_type=EXCLUDED.value_type,is_public=EXCLUDED.is_public,
record_status='A',updated_at=now()
"""
)
for key, value, kind, public in settings:
bind.execute(statement, {"key": key, "value": value, "kind": kind, "public": public})
def downgrade() -> None:
raise RuntimeError("Notification Center migration is forward-only")
VM1_app/codebase/backend/api-backend/alembic/versions/0009_chat_message_max_length.py
+33
View File
@@ -0,0 +1,33 @@
"""Seed configurable maximum chat message length.
Revision ID: 0009_chat_message_max
Revises: 0008_notifications_v1
Create Date: 2026-07-29
"""
from collections.abc import Sequence
from alembic import op
revision: str = "0009_chat_message_max"
down_revision: str | None = "0008_notifications_v1"
branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None
def upgrade() -> None:
op.execute(
"""
INSERT INTO han_app.app_settings
(setting_key, setting_value, value_type, is_public, description,
record_status, updated_at)
VALUES
('chat.message.max_length', '4000', 'integer', true,
'Maximum normalized client chat message length', 'A', now())
ON CONFLICT (setting_key) DO NOTHING
"""
)
def downgrade() -> None:
raise RuntimeError("Chat message maximum length migration is forward-only")
VM1_app/codebase/backend/api-backend/alembic/versions/0010_contact_map_dedup.py
+75
View File
@@ -0,0 +1,75 @@
"""Deduplicate contact mapping and skip no-op identity updates.
Revision ID: 0010_contact_map_dedup
Revises: 0009_chat_message_max
Create Date: 2026-08-05
"""
from collections.abc import Sequence
from alembic import op
revision: str = "0010_contact_map_dedup"
down_revision: str | None = "0009_chat_message_max"
branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None
def upgrade() -> None:
op.execute(
"""
CREATE OR REPLACE FUNCTION han_app.enqueue_contact_sync()
RETURNS trigger
LANGUAGE plpgsql
SECURITY INVOKER
SET search_path = han_app, pg_temp
AS $$
DECLARE
v_entity_id uuid;
v_task_type text;
v_dedup text;
BEGIN
IF current_setting('han.sync_suppress', true) = 'true' THEN
RETURN NEW;
END IF;
IF TG_TABLE_NAME = 'user_identities' THEN
IF TG_OP = 'UPDATE'
AND NEW.phone_number IS NOT DISTINCT FROM OLD.phone_number
AND NEW.record_status IS NOT DISTINCT FROM OLD.record_status THEN
RETURN NEW;
END IF;
v_entity_id := NEW.id;
v_task_type := CASE WHEN TG_OP = 'INSERT'
THEN 'contact.map_or_create' ELSE 'contact.update' END;
ELSE
v_entity_id := NEW.user_id;
v_task_type := CASE WHEN TG_OP = 'INSERT'
THEN 'contact.map_or_create' ELSE 'contact.update' END;
END IF;
IF v_task_type = 'contact.map_or_create' THEN
-- Identity and profile are inserted during the same bootstrap.
-- A stable key collapses both triggers into one logical task.
v_dedup := v_task_type || ':' || v_entity_id::text;
ELSE
v_dedup := v_task_type || ':' || v_entity_id::text || ':' ||
encode(public.digest(row_to_json(NEW)::text, 'sha256'), 'hex');
END IF;
INSERT INTO han_app.sync_queue
(id, task_type, entity_type, entity_id, dedup_key, payload_json,
status, attempt_count, next_attempt_at, created_at, updated_at)
VALUES
(gen_random_uuid(), v_task_type, 'contact', v_entity_id, v_dedup,
jsonb_build_object('entity_id', v_entity_id), 'pending', 0,
now(), now(), now())
ON CONFLICT (dedup_key) DO NOTHING;
RETURN NEW;
END;
$$
"""
)
def downgrade() -> None:
raise RuntimeError("Contact map-or-create deduplication migration is forward-only")
VM1_app/codebase/backend/api-backend/alembic/versions/0011_module07_contract.py
+188
View File
@@ -0,0 +1,188 @@
"""Expand module-07 queue and stage canonical Bitrix mapping.
Revision ID: 0011_module07_contract
Revises: 0010_contact_map_dedup
Create Date: 2026-08-06
"""
from collections.abc import Sequence
from alembic import op
revision: str = "0011_module07_contract"
down_revision: str | None = "0010_contact_map_dedup"
branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None
def upgrade() -> None:
# Expand first. Existing rows remain readable throughout the migration.
op.execute(
"""
ALTER TABLE han_app.sync_queue
ADD COLUMN IF NOT EXISTS locked_by varchar(128),
ADD COLUMN IF NOT EXISTS locked_until timestamptz,
ADD COLUMN IF NOT EXISTS lease_token uuid,
ADD COLUMN IF NOT EXISTS last_error_code varchar(64),
ADD COLUMN IF NOT EXISTS last_error_at timestamptz,
ADD COLUMN IF NOT EXISTS completed_at timestamptz,
ADD COLUMN IF NOT EXISTS cancel_reason varchar(255);
UPDATE han_app.sync_queue
SET status = CASE status
WHEN 'processing' THEN 'pending'
WHEN 'failed' THEN 'retry_wait'
ELSE status
END
WHERE status IN ('processing', 'failed');
ALTER TABLE han_app.sync_queue
DROP CONSTRAINT IF EXISTS sync_queue_status_check;
ALTER TABLE han_app.sync_queue
ADD CONSTRAINT sync_queue_status_check CHECK (
status IN ('pending','leased','processed','retry_wait','dead_letter','cancelled')
) NOT VALID;
ALTER TABLE han_app.sync_queue
VALIDATE CONSTRAINT sync_queue_status_check;
ALTER TABLE han_app.sync_queue
DROP CONSTRAINT IF EXISTS sync_queue_dedup_key_key;
DROP INDEX IF EXISTS han_app.ix_sync_queue_status_next;
CREATE INDEX IF NOT EXISTS ix_sync_queue_claim
ON han_app.sync_queue(status, next_attempt_at, created_at);
CREATE INDEX IF NOT EXISTS ix_sync_queue_expired_lease
ON han_app.sync_queue(locked_until) WHERE status = 'leased';
CREATE INDEX IF NOT EXISTS ix_sync_queue_entity_history
ON han_app.sync_queue(entity_type, entity_id, created_at DESC);
CREATE UNIQUE INDEX IF NOT EXISTS uq_sync_queue_active_dedup
ON han_app.sync_queue(dedup_key)
WHERE status IN ('pending','leased','retry_wait');
"""
)
# The bitrix-sync migration owns the canonical schema. If that migration
# already ran, copy and verify legacy rows here; otherwise its follow-up
# migration performs the same copy. The legacy table remains until the
# readers have switched and the contract migration is explicitly approved.
op.execute(
"""
DO $$
BEGIN
IF to_regclass('bitrix_sync.entity_external_mapping') IS NOT NULL THEN
INSERT INTO bitrix_sync.entity_external_mapping (
id, entity_type, entity_id, external_system, external_entity_type,
external_id, status, opened_at, created_at, updated_at
)
SELECT id, entity_type, entity_id, 'bitrix24', 'contact',
external_id, 'active', created_at, created_at, created_at
FROM han_app.entity_external_mapping
ON CONFLICT DO NOTHING;
IF EXISTS (
SELECT 1
FROM han_app.entity_external_mapping legacy
LEFT JOIN bitrix_sync.entity_external_mapping canonical
ON canonical.id = legacy.id
AND canonical.entity_type = legacy.entity_type
AND canonical.entity_id = legacy.entity_id
AND canonical.external_id = legacy.external_id
WHERE canonical.id IS NULL
) THEN
RAISE EXCEPTION 'canonical mapping verification failed';
END IF;
END IF;
END $$;
"""
)
op.execute(
"""
CREATE OR REPLACE FUNCTION han_app.enqueue_contact_sync()
RETURNS trigger
LANGUAGE plpgsql
SECURITY INVOKER
SET search_path = han_app, pg_temp
AS $$
DECLARE
v_user_id uuid;
v_task_type varchar(64);
v_reason varchar(64);
v_dedup varchar(255);
v_source_updated_at timestamptz;
BEGIN
IF current_setting('han.sync_suppress', true) = 'true' THEN
RETURN NEW;
END IF;
IF TG_TABLE_NAME = 'user_identities' THEN
v_user_id := NEW.id;
v_source_updated_at := NEW.updated_at;
IF TG_OP = 'INSERT' THEN
IF NEW.record_status <> 'A' THEN RETURN NEW; END IF;
v_task_type := 'contact.map_or_create';
v_reason := 'identity_created';
ELSIF OLD.record_status = 'A' AND NEW.record_status <> 'A' THEN
v_task_type := 'contact.deactivate';
v_reason := 'identity_deactivated';
ELSIF OLD.record_status <> 'A' AND NEW.record_status = 'A' THEN
v_task_type := 'contact.map_or_create';
v_reason := 'identity_reactivated';
ELSIF NEW.record_status = 'A'
AND NEW.phone_number IS DISTINCT FROM OLD.phone_number THEN
v_task_type := 'contact.update';
v_reason := 'identity_phone_changed';
ELSE
RETURN NEW;
END IF;
ELSE
v_user_id := NEW.user_id;
v_source_updated_at := NEW.updated_at;
IF TG_OP = 'INSERT' THEN
IF NEW.record_status <> 'A' THEN RETURN NEW; END IF;
v_task_type := 'contact.map_or_create';
v_reason := 'profile_created';
ELSIF OLD.record_status = 'A' AND NEW.record_status <> 'A' THEN
v_task_type := 'contact.deactivate';
v_reason := 'profile_deactivated';
ELSIF OLD.record_status <> 'A' AND NEW.record_status = 'A' THEN
v_task_type := 'contact.map_or_create';
v_reason := 'profile_reactivated';
ELSE
RETURN NEW;
END IF;
END IF;
v_dedup := v_task_type || ':' || v_user_id::text;
INSERT INTO han_app.sync_queue (
id, task_type, entity_type, entity_id, dedup_key, payload_json,
status, attempt_count, next_attempt_at, created_at, updated_at
) VALUES (
gen_random_uuid(), v_task_type, 'contact', v_user_id, v_dedup,
jsonb_build_object(
'schema_version', 1,
'user_id', v_user_id,
'reason', v_reason,
'source_updated_at', v_source_updated_at
),
'pending', 0, now(), now(), now()
)
ON CONFLICT (dedup_key)
WHERE status IN ('pending','leased','retry_wait')
DO UPDATE SET
payload_json = EXCLUDED.payload_json,
updated_at = now();
RETURN NEW;
END;
$$;
DROP TRIGGER IF EXISTS trg_profile_contact_sync ON han_app.client_profiles;
CREATE TRIGGER trg_profile_contact_sync
AFTER INSERT OR UPDATE OF record_status
ON han_app.client_profiles
FOR EACH ROW EXECUTE FUNCTION han_app.enqueue_contact_sync();
"""
)
def downgrade() -> None:
raise RuntimeError("Module-07 staged contract migration is forward-only")
VM1_app/codebase/backend/api-backend/alembic/versions/0012_safety_v2_checkpoint.py
+74
View File
@@ -0,0 +1,74 @@
"""Persist Message Safety v2 evidence and recovery locations.
Revision ID: 0012_safety_v2_checkpoint
Revises: 0011_module07_contract
Create Date: 2026-08-06
"""
from collections.abc import Sequence
from alembic import op
revision: str = "0012_safety_v2_checkpoint"
down_revision: str | None = "0011_module07_contract"
branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None
def upgrade() -> None:
op.execute(
"""
ALTER TABLE han_app.messages
ADD COLUMN IF NOT EXISTS safety_processing_mode varchar(16),
ADD COLUMN IF NOT EXISTS safety_config_version bigint,
ADD COLUMN IF NOT EXISTS safety_rules_version varchar(128);
ALTER TABLE han_app.message_attachments
ADD COLUMN IF NOT EXISTS quarantine_version_id varchar(1024),
ADD COLUMN IF NOT EXISTS quarantine_etag varchar(1024);
ALTER TABLE han_app.message_attachments
DROP CONSTRAINT IF EXISTS message_attachments_scan_status_check;
ALTER TABLE han_app.message_attachments
ADD CONSTRAINT message_attachments_scan_status_check
CHECK (scan_status IN ('pending','clean','bypassed','infected','failed')) NOT VALID;
ALTER TABLE han_app.message_attachments
VALIDATE CONSTRAINT message_attachments_scan_status_check;
ALTER TABLE han_app.safety_tasks
ADD COLUMN IF NOT EXISTS poll_location varchar(1024),
ADD COLUMN IF NOT EXISTS processing_mode varchar(16),
ADD COLUMN IF NOT EXISTS config_version bigint,
ADD COLUMN IF NOT EXISTS rules_version varchar(128),
ADD COLUMN IF NOT EXISTS expires_at timestamptz;
UPDATE han_app.safety_tasks
SET poll_location = '/internal/safety/v2/messages/tasks/' || task_id,
expires_at = deadline_at
WHERE poll_location IS NULL OR expires_at IS NULL;
ALTER TABLE han_app.safety_tasks
ALTER COLUMN poll_location SET NOT NULL,
ALTER COLUMN expires_at SET NOT NULL;
"""
)
# Notification uploads use the same safety contract.
op.execute(
"""
ALTER TABLE han_app.client_upload_drafts
ADD COLUMN IF NOT EXISTS quarantine_version_id varchar(1024),
ADD COLUMN IF NOT EXISTS quarantine_etag varchar(1024),
ADD COLUMN IF NOT EXISTS safety_processing_mode varchar(16),
ADD COLUMN IF NOT EXISTS safety_config_version bigint,
ADD COLUMN IF NOT EXISTS safety_rules_version varchar(128);
ALTER TABLE han_app.client_upload_drafts
DROP CONSTRAINT IF EXISTS client_upload_drafts_scan_status_check;
ALTER TABLE han_app.client_upload_drafts
ADD CONSTRAINT client_upload_drafts_scan_status_check
CHECK (scan_status IN ('pending','clean','bypassed','infected','failed')) NOT VALID;
ALTER TABLE han_app.client_upload_drafts
VALIDATE CONSTRAINT client_upload_drafts_scan_status_check;
"""
)
def downgrade() -> None:
raise RuntimeError("Safety v2 checkpoint migration is forward-only")
VM1_app/codebase/backend/api-backend/app/__init__.py
+1
View File
@@ -0,0 +1 @@
"""HAN Chat API backend."""
VM1_app/codebase/backend/api-backend/app/auth.py
+110
View File
@@ -0,0 +1,110 @@
import asyncio
import time
from dataclasses import dataclass
from typing import Any
import httpx
import jwt
import phonenumbers
from jwt import ExpiredSignatureError, InvalidTokenError, PyJWK
from app.settings import Settings
class AuthError(Exception):
def __init__(self, code: str = "unauthorized") -> None:
self.code = code
@dataclass(frozen=True, slots=True)
class Principal:
subject: str
phone_number: str | None
claims: dict[str, Any]
class JWKSValidator:
def __init__(self, settings: Settings, http: httpx.AsyncClient) -> None:
self.settings = settings
self.http = http
self._keys: dict[str, PyJWK] = {}
self._loaded_at = 0.0
self._lock = asyncio.Lock()
@property
def has_keys(self) -> bool:
return bool(self._keys)
async def refresh(self) -> None:
async with self._lock:
discovery_url = (
f"{str(self.settings.keycloak_internal_url).rstrip('/')}"
f"/realms/{self.settings.keycloak_realm}/.well-known/openid-configuration"
)
discovery = (await self.http.get(discovery_url, timeout=3)).raise_for_status().json()
jwks_uri = discovery["jwks_uri"]
public_prefix = str(self.settings.keycloak_public_url).rstrip("/")
internal_prefix = str(self.settings.keycloak_internal_url).rstrip("/")
if jwks_uri.startswith(public_prefix):
jwks_uri = internal_prefix + jwks_uri[len(public_prefix) :]
payload = (await self.http.get(jwks_uri, timeout=3)).raise_for_status().json()
self._keys = {
key["kid"]: PyJWK.from_dict(key)
for key in payload.get("keys", [])
if key.get("kid") and key.get("kty") == "RSA"
}
self._loaded_at = time.monotonic()
async def validate(self, token: str) -> Principal:
try:
header = jwt.get_unverified_header(token)
except InvalidTokenError as exc:
raise AuthError() from exc
if header.get("alg") != "RS256" or not header.get("kid"):
raise AuthError()
kid = str(header["kid"])
stale = time.monotonic() - self._loaded_at > self.settings.jwks_cache_ttl_seconds
if stale or kid not in self._keys:
try:
await self.refresh()
except (httpx.HTTPError, KeyError, ValueError):
if (
kid not in self._keys
or time.monotonic() - self._loaded_at > self.settings.jwks_stale_grace_seconds
):
raise AuthError() from None
key = self._keys.get(kid)
if key is None:
raise AuthError()
try:
claims = jwt.decode(
token,
key.key,
algorithms=["RS256"],
audience=self.settings.keycloak_audience,
issuer=self.settings.issuer,
options={"require": ["exp", "sub"]},
leeway=30,
)
except ExpiredSignatureError as exc:
raise AuthError("token_expired") from exc
except InvalidTokenError as exc:
raise AuthError() from exc
subject = claims.get("sub")
if not isinstance(subject, str) or not subject:
raise AuthError()
return Principal(subject, canonical_phone(claims), claims)
def canonical_phone(claims: dict[str, Any]) -> str | None:
for name in ("phone_number", "preferred_username"):
value = claims.get(name)
if not isinstance(value, str):
continue
try:
number = phonenumbers.parse(value, None)
except phonenumbers.NumberParseException:
continue
if phonenumbers.is_valid_number(number) and value.startswith("+"):
return phonenumbers.format_number(number, phonenumbers.PhoneNumberFormat.E164)
return None
VM1_app/codebase/backend/api-backend/app/chat_settings.py
+21
View File
@@ -0,0 +1,21 @@
from collections.abc import Mapping
CHAT_MESSAGE_MAX_LENGTH_KEY = "chat.message.max_length"
CHAT_MESSAGE_TRANSPORT_MAX_LENGTH = 10_000
def validate_chat_settings(values: Mapping[str, str]) -> None:
raw = values.get(CHAT_MESSAGE_MAX_LENGTH_KEY)
if raw is None:
return
try:
value = int(raw)
except (TypeError, ValueError) as error:
raise ValueError(f"{CHAT_MESSAGE_MAX_LENGTH_KEY}: integer value expected") from error
if str(value) != raw:
raise ValueError(f"{CHAT_MESSAGE_MAX_LENGTH_KEY}: canonical integer value expected")
if not 1 <= value <= CHAT_MESSAGE_TRANSPORT_MAX_LENGTH:
raise ValueError(
f"{CHAT_MESSAGE_MAX_LENGTH_KEY}: value must be between 1 "
f"and {CHAT_MESSAGE_TRANSPORT_MAX_LENGTH}"
)
VM1_app/codebase/backend/api-backend/app/cli/__init__.py
+1
View File
@@ -0,0 +1 @@
"""Operational command-line entry points."""
VM1_app/codebase/backend/api-backend/app/cli/seed_settings.py
+127
View File
@@ -0,0 +1,127 @@
from __future__ import annotations
import argparse
import asyncio
from pathlib import Path
from typing import Any
import yaml
from sqlalchemy import func, or_
from sqlalchemy.dialects.postgresql import insert
from app.chat_settings import CHAT_MESSAGE_MAX_LENGTH_KEY, validate_chat_settings
from app.db import AppSetting, Database
from app.otp_settings import OTP_SETTING_KEYS, validate_otp_settings
from app.settings import get_settings
VALUE_TYPES = {"boolean", "integer", "string", "string_list"}
def load_seed(path: Path) -> list[dict[str, Any]]:
document = yaml.safe_load(path.read_text(encoding="utf-8"))
if not isinstance(document, dict) or document.get("schema_version") != 1:
raise ValueError("settings file must have schema_version: 1")
settings = document.get("settings")
if not isinstance(settings, dict) or not settings:
raise ValueError("settings file must contain a non-empty settings mapping")
rows: list[dict[str, Any]] = []
for key, raw in settings.items():
if not isinstance(key, str) or not key.strip():
raise ValueError("setting keys must be non-empty strings")
if not isinstance(raw, dict):
raise ValueError(f"{key}: setting must be a mapping")
value_type = raw.get("type")
if value_type not in VALUE_TYPES:
raise ValueError(f"{key}: unsupported type {value_type!r}")
if key in OTP_SETTING_KEYS and value_type != "integer":
raise ValueError(f"{key}: type must be integer")
if key == CHAT_MESSAGE_MAX_LENGTH_KEY and value_type != "integer":
raise ValueError(f"{key}: type must be integer")
if not isinstance(raw.get("public"), bool):
raise ValueError(f"{key}: public must be a boolean")
if key in OTP_SETTING_KEYS and raw["public"]:
raise ValueError(f"{key}: OTP setting must not be public")
if key == CHAT_MESSAGE_MAX_LENGTH_KEY and not raw["public"]:
raise ValueError(f"{key}: setting must be public")
description = raw.get("description")
if description is not None and not isinstance(description, str):
raise ValueError(f"{key}: description must be a string")
rows.append(
{
"setting_key": key,
"setting_value": serialize_value(key, value_type, raw.get("value")),
"value_type": value_type,
"is_public": raw["public"],
"description": description,
"record_status": "A",
}
)
validate_otp_settings({row["setting_key"]: row["setting_value"] for row in rows})
validate_chat_settings({row["setting_key"]: row["setting_value"] for row in rows})
return rows
def serialize_value(key: str, value_type: str, value: Any) -> str:
if value_type == "boolean":
if not isinstance(value, bool):
raise ValueError(f"{key}: boolean value expected")
return str(value).lower()
if value_type == "integer":
if not isinstance(value, int) or isinstance(value, bool):
raise ValueError(f"{key}: integer value expected")
return str(value)
if value_type == "string_list":
if isinstance(value, list) and all(isinstance(item, str) for item in value):
return ",".join(value)
if isinstance(value, str):
return value
raise ValueError(f"{key}: string or list of strings expected")
if not isinstance(value, str):
raise ValueError(f"{key}: string value expected")
return value
async def seed(path: Path) -> int:
rows = load_seed(path)
database = Database(get_settings().database_url)
try:
async with database.sessions() as session:
for row in rows:
statement = insert(AppSetting).values(**row)
excluded = statement.excluded
statement = statement.on_conflict_do_update(
index_elements=[AppSetting.setting_key],
set_={
"setting_value": excluded.setting_value,
"value_type": excluded.value_type,
"is_public": excluded.is_public,
"description": excluded.description,
"record_status": "A",
"updated_at": func.now(),
},
where=or_(
AppSetting.setting_value.is_distinct_from(excluded.setting_value),
AppSetting.value_type.is_distinct_from(excluded.value_type),
AppSetting.is_public.is_distinct_from(excluded.is_public),
AppSetting.description.is_distinct_from(excluded.description),
AppSetting.record_status != "A",
),
)
await session.execute(statement)
await session.commit()
finally:
await database.close()
return len(rows)
def main() -> None:
parser = argparse.ArgumentParser(description="Idempotently seed application settings")
parser.add_argument("--file", required=True, type=Path)
args = parser.parse_args()
count = asyncio.run(seed(args.file))
print(f"Application settings seeded: {count}")
if __name__ == "__main__":
main()
VM1_app/codebase/backend/api-backend/app/cli/validate_settings.py
+40
View File
@@ -0,0 +1,40 @@
from __future__ import annotations
import asyncio
from sqlalchemy import select
from app.db import AppSetting, Database
from app.services import REQUIRED_SETTINGS
from app.settings import get_settings
async def validate() -> int:
database = Database(get_settings().database_url)
try:
async with database.sessions() as session:
active_keys = set(
(
await session.execute(
select(AppSetting.setting_key).where(AppSetting.record_status == "A")
)
)
.scalars()
.all()
)
finally:
await database.close()
missing = sorted(REQUIRED_SETTINGS - active_keys)
if missing:
raise RuntimeError(f"Mandatory application settings are missing: {', '.join(missing)}")
return len(REQUIRED_SETTINGS)
def main() -> None:
count = asyncio.run(validate())
print(f"Mandatory application settings validated: {count}")
if __name__ == "__main__":
main()
VM1_app/codebase/backend/api-backend/app/db.py
+435
View File
@@ -0,0 +1,435 @@
import uuid
from collections.abc import AsyncIterator
from datetime import datetime
from typing import Any
from sqlalchemy import (
JSON,
BigInteger,
Boolean,
CheckConstraint,
DateTime,
ForeignKey,
Index,
Integer,
String,
Text,
UniqueConstraint,
func,
)
from sqlalchemy.dialects.postgresql import INET, JSONB, UUID
from sqlalchemy.ext.asyncio import (
AsyncEngine,
AsyncSession,
async_sessionmaker,
)
from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column
from app.postgres import create_postgres_engine
SCHEMA = "han_app"
class Base(DeclarativeBase):
type_annotation_map = {dict[str, Any]: JSON}
class BitrixBase(DeclarativeBase):
"""Models owned by bitrix-sync, excluded from han_app create_all."""
class Common:
id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
record_status: Mapped[str] = mapped_column(String(1), default="A", server_default="A")
status_changed_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
status_change_reason: Mapped[str | None] = mapped_column(String(255))
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
updated_at: Mapped[datetime] = mapped_column(
DateTime(timezone=True), server_default=func.now(), onupdate=func.now()
)
updater_user_id: Mapped[uuid.UUID | None] = mapped_column(UUID(as_uuid=True))
class UserIdentity(Common, Base):
__tablename__ = "user_identities"
__table_args__ = (Index("ix_user_identities_phone", "phone_number"), {"schema": SCHEMA})
keycloak_sub: Mapped[str] = mapped_column(String(255), unique=True)
phone_number: Mapped[str] = mapped_column(String(32))
last_login_at: Mapped[datetime] = mapped_column(DateTime(timezone=True))
class ClientProfile(Common, Base):
__tablename__ = "client_profiles"
__table_args__ = ({"schema": SCHEMA},)
user_id: Mapped[uuid.UUID] = mapped_column(
ForeignKey(f"{SCHEMA}.user_identities.id", ondelete="RESTRICT"), unique=True
)
bitrix_contact_id: Mapped[str | None] = mapped_column(String(64))
full_name: Mapped[str | None] = mapped_column(String(255))
citizenship: Mapped[str | None] = mapped_column(String(128))
russian_phone: Mapped[str | None] = mapped_column(String(32))
foreign_phone: Mapped[str | None] = mapped_column(String(32))
email: Mapped[str | None] = mapped_column(String(320))
source_updated_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
class UserConsent(Common, Base):
__tablename__ = "user_consents"
__table_args__ = (
UniqueConstraint("user_id", "consent_type", "document_version"),
CheckConstraint("consent_type IN ('personal_data','user_agreement','marketing')"),
{"schema": SCHEMA},
)
user_id: Mapped[uuid.UUID] = mapped_column(
ForeignKey(f"{SCHEMA}.user_identities.id", ondelete="RESTRICT")
)
ux_session_id: Mapped[uuid.UUID | None] = mapped_column(UUID(as_uuid=True))
consent_type: Mapped[str] = mapped_column(String(32))
document_version: Mapped[str] = mapped_column(String(64))
accepted: Mapped[bool] = mapped_column(Boolean)
accepted_at: Mapped[datetime] = mapped_column(DateTime(timezone=True))
client_ip: Mapped[str | None] = mapped_column(INET)
user_agent_hash: Mapped[str | None] = mapped_column(String(64))
device_json: Mapped[dict[str, Any]] = mapped_column(JSONB, default=dict)
class UxSession(Common, Base):
__tablename__ = "ux_sessions"
__table_args__ = (
CheckConstraint("start_reason IN ('first_launch','cold_start','idle_timeout')"),
Index("ix_ux_sessions_user_started", "user_id", "started_at"),
{"schema": SCHEMA},
)
user_id: Mapped[uuid.UUID] = mapped_column(
ForeignKey(f"{SCHEMA}.user_identities.id", ondelete="RESTRICT")
)
start_reason: Mapped[str] = mapped_column(String(32))
platform: Mapped[str] = mapped_column(String(32))
app_version: Mapped[str] = mapped_column(String(64))
device_id_hash: Mapped[str | None] = mapped_column(String(64))
device_id: Mapped[str | None] = mapped_column(String(255))
started_at: Mapped[datetime] = mapped_column(DateTime(timezone=True))
class Dialog(Common, Base):
__tablename__ = "dialogs"
__table_args__ = (
CheckConstraint("status IN ('open','waiting_for_company','waiting_for_client','closed')"),
Index("ix_dialogs_user_updated", "user_id", "updated_at", "id"),
{"schema": SCHEMA},
)
user_id: Mapped[uuid.UUID] = mapped_column(
ForeignKey(f"{SCHEMA}.user_identities.id", ondelete="RESTRICT")
)
status: Mapped[str] = mapped_column(String(32), default="open")
last_message_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
closed_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
class Message(Common, Base):
__tablename__ = "messages"
__table_args__ = (
CheckConstraint("sender_type IN ('client','company')"),
CheckConstraint("content_kind IN ('text','file')"),
CheckConstraint("safety_status IN ('pending','allowed','blocked','needs_review')"),
CheckConstraint(
"delivery_status IN ('accepted','processing','delivered','rejected','failed')"
),
Index("ix_messages_dialog_created", "dialog_id", "created_at", "id"),
UniqueConstraint("dialog_id", "client_idempotency_key"),
{"schema": SCHEMA},
)
dialog_id: Mapped[uuid.UUID] = mapped_column(
ForeignKey(f"{SCHEMA}.dialogs.id", ondelete="RESTRICT")
)
sender_type: Mapped[str] = mapped_column(String(16))
content_kind: Mapped[str] = mapped_column(String(16))
text: Mapped[str] = mapped_column(Text, default="")
safety_status: Mapped[str] = mapped_column(String(16))
safety_processing_mode: Mapped[str | None] = mapped_column(String(16))
safety_config_version: Mapped[int | None] = mapped_column(BigInteger)
safety_rules_version: Mapped[str | None] = mapped_column(String(128))
delivery_status: Mapped[str] = mapped_column(String(16))
external_message_id: Mapped[str | None] = mapped_column(String(255))
client_idempotency_key: Mapped[str | None] = mapped_column(String(128))
occurred_at: Mapped[datetime] = mapped_column(DateTime(timezone=True))
class MessageAttachment(Common, Base):
__tablename__ = "message_attachments"
__table_args__ = (
CheckConstraint("direction IN ('client_upload','company_inbound')"),
CheckConstraint("scan_status IN ('pending','clean','bypassed','infected','failed')"),
CheckConstraint("size_bytes > 0"),
{"schema": SCHEMA},
)
dialog_id: Mapped[uuid.UUID] = mapped_column(ForeignKey(f"{SCHEMA}.dialogs.id"))
message_id: Mapped[uuid.UUID | None] = mapped_column(
ForeignKey(f"{SCHEMA}.messages.id"), unique=True
)
owner_user_id: Mapped[uuid.UUID] = mapped_column(ForeignKey(f"{SCHEMA}.user_identities.id"))
direction: Mapped[str] = mapped_column(String(32))
original_file_name: Mapped[str] = mapped_column(String(255))
safe_file_name: Mapped[str] = mapped_column(String(255))
mime_type: Mapped[str] = mapped_column(String(128))
size_bytes: Mapped[int] = mapped_column(BigInteger)
checksum_sha256: Mapped[str | None] = mapped_column(String(64))
scan_status: Mapped[str] = mapped_column(String(16), default="pending")
storage_bucket: Mapped[str] = mapped_column(String(255))
object_key: Mapped[str] = mapped_column(String(1024))
quarantine_object_key: Mapped[str | None] = mapped_column(String(1024))
quarantine_version_id: Mapped[str | None] = mapped_column(String(1024))
quarantine_etag: Mapped[str | None] = mapped_column(String(1024))
upload_expires_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
completed_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
class Document(Common, Base):
__tablename__ = "documents"
__table_args__ = (UniqueConstraint("storage_bucket", "object_key"), {"schema": SCHEMA})
user_id: Mapped[uuid.UUID] = mapped_column(ForeignKey(f"{SCHEMA}.user_identities.id"))
name: Mapped[str] = mapped_column(String(255))
mime_type: Mapped[str] = mapped_column(String(128))
size_bytes: Mapped[int] = mapped_column(BigInteger)
checksum_sha256: Mapped[str] = mapped_column(String(64))
storage_bucket: Mapped[str] = mapped_column(String(255))
object_key: Mapped[str] = mapped_column(String(1024))
sent_at: Mapped[datetime] = mapped_column(DateTime(timezone=True))
class SafetyTask(Base):
__tablename__ = "safety_tasks"
__table_args__ = ({"schema": SCHEMA},)
id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
task_id: Mapped[str] = mapped_column(String(255), unique=True)
poll_location: Mapped[str] = mapped_column(String(1024))
message_id: Mapped[uuid.UUID] = mapped_column(ForeignKey(f"{SCHEMA}.messages.id"), unique=True)
attachment_id: Mapped[uuid.UUID | None] = mapped_column(UUID(as_uuid=True))
quarantine_object_key: Mapped[str | None] = mapped_column(String(1024))
status: Mapped[str] = mapped_column(String(16))
processing_mode: Mapped[str | None] = mapped_column(String(16))
config_version: Mapped[int | None] = mapped_column(BigInteger)
rules_version: Mapped[str | None] = mapped_column(String(128))
expires_at: Mapped[datetime] = mapped_column(DateTime(timezone=True))
deadline_at: Mapped[datetime] = mapped_column(DateTime(timezone=True))
next_poll_at: Mapped[datetime] = mapped_column(DateTime(timezone=True))
attempt_count: Mapped[int] = mapped_column(Integer, default=0)
last_error_code: Mapped[str | None] = mapped_column(String(64))
locked_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
locked_by: Mapped[str | None] = mapped_column(String(128))
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
class DeliveryOutbox(Base):
__tablename__ = "delivery_outbox"
__table_args__ = ({"schema": SCHEMA},)
id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
message_id: Mapped[uuid.UUID] = mapped_column(ForeignKey(f"{SCHEMA}.messages.id"), unique=True)
external_chat_id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True))
payload_json: Mapped[dict[str, Any]] = mapped_column(JSON)
status: Mapped[str] = mapped_column(String(16), default="pending")
attempt_count: Mapped[int] = mapped_column(Integer, default=0)
next_attempt_at: Mapped[datetime] = mapped_column(DateTime(timezone=True))
last_error_code: Mapped[str | None] = mapped_column(String(64))
locked_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
locked_by: Mapped[str | None] = mapped_column(String(128))
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
class IdempotencyRecord(Base):
__tablename__ = "idempotency_records"
__table_args__ = (
UniqueConstraint("scope", "user_id", "idempotency_key"),
{"schema": SCHEMA},
)
id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
scope: Mapped[str] = mapped_column(String(128))
user_id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True))
idempotency_key: Mapped[str] = mapped_column(String(128))
request_fingerprint: Mapped[str] = mapped_column(String(64))
status: Mapped[str] = mapped_column(String(16))
response_status: Mapped[int | None] = mapped_column(Integer)
response_body_json: Mapped[dict[str, Any] | None] = mapped_column(JSON)
resource_type: Mapped[str | None] = mapped_column(String(64))
resource_id: Mapped[uuid.UUID | None] = mapped_column(UUID(as_uuid=True))
expires_at: Mapped[datetime] = mapped_column(DateTime(timezone=True))
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
class OpenLinesInboxReceipt(Base):
__tablename__ = "openlines_inbox_receipts"
__table_args__ = (
UniqueConstraint("event_id"),
UniqueConstraint("external_chat_id", "bitrix_message_id"),
{"schema": SCHEMA},
)
id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
event_id: Mapped[str] = mapped_column(String(255))
external_chat_id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True))
bitrix_message_id: Mapped[str | None] = mapped_column(String(255))
event_type: Mapped[str] = mapped_column(String(32))
payload_fingerprint: Mapped[str] = mapped_column(String(64))
status: Mapped[str] = mapped_column(String(16))
message_id: Mapped[uuid.UUID | None] = mapped_column(UUID(as_uuid=True))
last_error_code: Mapped[str | None] = mapped_column(String(64))
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
class AuditEvent(Base):
__tablename__ = "audit_events"
__table_args__ = ({"schema": SCHEMA},)
id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
event_type: Mapped[str] = mapped_column(String(128))
actor_type: Mapped[str] = mapped_column(String(32))
user_id: Mapped[uuid.UUID | None] = mapped_column(UUID(as_uuid=True))
ux_session_id: Mapped[uuid.UUID | None] = mapped_column(UUID(as_uuid=True))
request_id: Mapped[str] = mapped_column(String(64))
trace_id: Mapped[str | None] = mapped_column(String(64))
resource_type: Mapped[str | None] = mapped_column(String(64))
resource_id: Mapped[uuid.UUID | None] = mapped_column(UUID(as_uuid=True))
user_agent_hash: Mapped[str | None] = mapped_column(String(64))
outcome: Mapped[str] = mapped_column(String(32))
metadata_json: Mapped[dict[str, Any]] = mapped_column(JSON, default=dict)
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
class AppSetting(Base):
__tablename__ = "app_settings"
__table_args__ = ({"schema": SCHEMA},)
setting_key: Mapped[str] = mapped_column(String(255), primary_key=True)
setting_value: Mapped[str] = mapped_column(Text)
value_type: Mapped[str] = mapped_column(String(32))
is_public: Mapped[bool] = mapped_column(Boolean, default=False)
description: Mapped[str | None] = mapped_column(Text)
record_status: Mapped[str] = mapped_column(String(1), default="A")
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
class TextResource(Common, Base):
__tablename__ = "text_resources"
__table_args__ = (
UniqueConstraint("mnemonic", "locale"),
{"schema": SCHEMA},
)
mnemonic: Mapped[str] = mapped_column(String(255))
locale: Mapped[str] = mapped_column(String(16), default="ru")
text_value: Mapped[str] = mapped_column(Text)
sort_order: Mapped[int] = mapped_column(Integer, default=0)
class PopularQuestion(Common, Base):
__tablename__ = "popular_questions"
__table_args__ = (
UniqueConstraint("mnemonic", "locale"),
{"schema": SCHEMA},
)
mnemonic: Mapped[str] = mapped_column(String(255))
locale: Mapped[str] = mapped_column(String(16), default="ru")
question_text: Mapped[str] = mapped_column(Text)
sort_order: Mapped[int] = mapped_column(Integer, default=0)
class SyncQueue(Base):
__tablename__ = "sync_queue"
__table_args__ = (
CheckConstraint(
"status IN ('pending','leased','processed','retry_wait','dead_letter','cancelled')"
),
Index("ix_sync_queue_claim", "status", "next_attempt_at", "created_at"),
Index("ix_sync_queue_entity_history", "entity_type", "entity_id", "created_at"),
{"schema": SCHEMA},
)
id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
task_type: Mapped[str] = mapped_column(String(64))
entity_type: Mapped[str] = mapped_column(String(64))
entity_id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True))
dedup_key: Mapped[str] = mapped_column(String(255))
payload_json: Mapped[dict[str, Any]] = mapped_column(JSON)
status: Mapped[str] = mapped_column(String(16), default="pending")
attempt_count: Mapped[int] = mapped_column(Integer, default=0)
next_attempt_at: Mapped[datetime] = mapped_column(
DateTime(timezone=True), server_default=func.now()
)
locked_by: Mapped[str | None] = mapped_column(String(128))
locked_until: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
lease_token: Mapped[uuid.UUID | None] = mapped_column(UUID(as_uuid=True))
last_error_code: Mapped[str | None] = mapped_column(String(64))
last_error_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
completed_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
cancel_reason: Mapped[str | None] = mapped_column(String(255))
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
class EntityExternalMapping(Base):
__tablename__ = "entity_external_mapping"
__table_args__ = (UniqueConstraint("entity_type", "entity_id"), {"schema": SCHEMA})
id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
entity_type: Mapped[str] = mapped_column(String(64))
entity_id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True))
external_id: Mapped[str] = mapped_column(String(128))
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
class BitrixEntityExternalMapping(BitrixBase):
__tablename__ = "entity_external_mapping"
__table_args__ = ({"schema": "bitrix_sync"},)
id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
entity_type: Mapped[str] = mapped_column(String(64))
entity_id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True))
external_system: Mapped[str] = mapped_column(String(32), default="bitrix24")
external_entity_type: Mapped[str] = mapped_column(String(32), default="contact")
external_id: Mapped[str] = mapped_column(String(128))
status: Mapped[str] = mapped_column(String(16), default="active")
opened_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
closed_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
close_reason: Mapped[str | None] = mapped_column(String(64))
workflow_id: Mapped[uuid.UUID | None] = mapped_column(UUID(as_uuid=True))
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
Index(
"uq_sync_queue_active_dedup",
SyncQueue.dedup_key,
unique=True,
postgresql_where=SyncQueue.status.in_(["pending", "leased", "retry_wait"]),
)
Index(
"ix_sync_queue_expired_lease",
SyncQueue.locked_until,
postgresql_where=SyncQueue.status == "leased",
)
Index(
"uq_external_mapping_active_entity",
BitrixEntityExternalMapping.external_system,
BitrixEntityExternalMapping.entity_type,
BitrixEntityExternalMapping.entity_id,
unique=True,
postgresql_where=BitrixEntityExternalMapping.status == "active",
)
Index(
"uq_external_mapping_active_external",
BitrixEntityExternalMapping.external_system,
BitrixEntityExternalMapping.external_entity_type,
BitrixEntityExternalMapping.external_id,
unique=True,
postgresql_where=BitrixEntityExternalMapping.status == "active",
)
class Database:
def __init__(self, url: str) -> None:
self.engine: AsyncEngine = create_postgres_engine(url, pool_pre_ping=True)
self.sessions = async_sessionmaker(self.engine, expire_on_commit=False)
async def session(self) -> AsyncIterator[AsyncSession]:
async with self.sessions() as session:
yield session
async def close(self) -> None:
await self.engine.dispose()
VM1_app/codebase/backend/api-backend/app/integrations.py
+460
View File
@@ -0,0 +1,460 @@
import asyncio
import hashlib
import ipaddress
import json
import socket
import time
import uuid
from dataclasses import dataclass
from datetime import datetime
from typing import Any
from urllib.parse import urlparse
import boto3
import httpx
from botocore.config import Config
from redis.asyncio import Redis
from app.settings import Settings
RATE_LIMIT_LUA = """
local current = redis.call('INCR', KEYS[1])
if current == 1 then redis.call('EXPIRE', KEYS[1], ARGV[1]) end
local ttl = redis.call('TTL', KEYS[1])
return {current, ttl}
"""
class DependencyFailure(Exception):
def __init__(
self,
code: str = "dependency_unavailable",
timeout: bool = False,
*,
terminal: bool = False,
retryable: bool = True,
) -> None:
super().__init__(code)
self.code = code
self.timeout = timeout
self.terminal = terminal
self.retryable = retryable
@dataclass(slots=True)
class CircuitBreaker:
threshold: int
open_seconds: float
failures: int = 0
opened_at: float | None = None
def allow(self) -> bool:
if self.opened_at is None:
return True
if time.monotonic() - self.opened_at >= self.open_seconds:
self.opened_at = None
self.failures = max(0, self.threshold - 1)
return True
return False
def success(self) -> None:
self.failures = 0
self.opened_at = None
def failure(self) -> None:
self.failures += 1
if self.failures >= self.threshold:
self.opened_at = time.monotonic()
class RateLimiter:
def __init__(self, redis: Redis) -> None:
self.redis = redis
async def consume(self, key: str, limit: int, window: int) -> int:
try:
count, ttl = await self.redis.eval(RATE_LIMIT_LUA, 1, key, window)
except Exception as exc:
raise DependencyFailure() from exc
if int(count) > limit:
return max(1, int(ttl))
return 0
@staticmethod
def key(identity_type: str, identity: str, route: str, window: int) -> str:
safe_identity = hashlib.sha256(identity.encode()).hexdigest()[:32]
bucket = int(time.time()) // window
return f"han:api:rl:{identity_type}:{safe_identity}:{route}:{bucket}"
class RedisIdempotency:
def __init__(self, redis: Redis) -> None:
self.redis = redis
async def get(self, scope: str, user_id: uuid.UUID, key: str) -> dict[str, Any] | None:
key_hash = hashlib.sha256(key.encode()).hexdigest()
raw = await self.redis.get(f"han:api:idem:{scope}:{user_id}:{key_hash}")
return json.loads(raw) if raw else None
async def put(self, scope: str, user_id: uuid.UUID, key: str, value: dict[str, Any]) -> None:
key_hash = hashlib.sha256(key.encode()).hexdigest()
await self.redis.set(
f"han:api:idem:{scope}:{user_id}:{key_hash}",
json.dumps(value, separators=(",", ":"), default=str),
ex=86400,
)
class SafetyClient:
def __init__(self, settings: Settings, http: httpx.AsyncClient) -> None:
self.settings = settings
self.http = http
self.breaker = CircuitBreaker(
settings.message_safety_circuit_failure_threshold,
settings.message_safety_circuit_open_sec,
)
async def check(self, payload: dict[str, Any], request_id: str) -> dict[str, Any]:
return await self._call(
"POST",
f"{self.settings.message_safety_api_prefix}/messages/check",
request_id,
json=payload,
timeout=self.settings.message_safety_post_timeout_sec,
)
async def poll(self, location: str, request_id: str) -> dict[str, Any]:
path = self._poll_path(location)
return await self._call(
"GET",
path,
request_id,
timeout=2,
)
def _poll_path(self, location: str) -> str:
expected_prefix = f"{self.settings.message_safety_api_prefix}/messages/tasks/"
parsed = urlparse(location)
if parsed.scheme or parsed.netloc or parsed.query or parsed.fragment:
raise DependencyFailure("invalid_safety_location", terminal=True, retryable=False)
if not parsed.path.startswith(expected_prefix):
raise DependencyFailure("invalid_safety_location", terminal=True, retryable=False)
task_id = parsed.path.removeprefix(expected_prefix)
try:
uuid.UUID(task_id)
except ValueError as exc:
raise DependencyFailure(
"invalid_safety_location", terminal=True, retryable=False
) from exc
return parsed.path
async def _call(self, method: str, path: str, request_id: str, **kwargs: Any) -> dict[str, Any]:
if not self.breaker.allow():
raise DependencyFailure()
headers = {
"X-Service-Token": self.settings.message_safety_service_token.get_secret_value(),
"X-Request-ID": request_id,
}
try:
response = await self.http.request(
method,
f"{str(self.settings.message_safety_url).rstrip('/')}{path}",
headers=headers,
**kwargs,
)
except httpx.TimeoutException as exc:
self.breaker.failure()
raise DependencyFailure(timeout=True) from exc
except httpx.HTTPError as exc:
self.breaker.failure()
raise DependencyFailure() from exc
if response.status_code >= 500:
self.breaker.failure()
code, terminal, retryable = "dependency_unavailable", False, True
try:
details = response.json().get("error", {}).get("details", {})
code = response.json().get("error", {}).get("code", code)
terminal = details.get("terminal") is True
retryable = details.get("retryable") is not False
except (AttributeError, ValueError):
pass
raise DependencyFailure(code, terminal=terminal, retryable=retryable)
if response.status_code not in (200, 202, 403):
if response.status_code == 401:
self.breaker.failure()
code = "safety_request_rejected"
try:
code = response.json().get("error", {}).get("code", code)
except (AttributeError, ValueError):
pass
retryable = response.status_code in (404, 429)
raise DependencyFailure(
code,
terminal=not retryable,
retryable=retryable,
)
try:
body = response.json()
except ValueError as exc:
self.breaker.failure()
raise DependencyFailure() from exc
if not isinstance(body, dict):
self.breaker.failure()
raise DependencyFailure()
status = response.status_code
expected_verdict = {200: "allow", 202: "pending", 403: "deny"}[status]
if (
body.get("verdict") != expected_verdict
or body.get("processing_mode") not in ("standard", "mock")
or type(body.get("config_version")) is not int
or not body.get("rules_version")
):
self.breaker.failure()
raise DependencyFailure("invalid_safety_response")
if status == 202:
location = response.headers.get("Location")
retry_after = response.headers.get("Retry-After")
if (
body["processing_mode"] != "standard"
or not location
or not retry_after
or not body.get("task_id")
or not body.get("expires_at")
or type(body.get("poll_after_ms")) is not int
):
self.breaker.failure()
raise DependencyFailure("invalid_safety_response")
try:
location_task_id = self._poll_path(location).rsplit("/", 1)[-1]
except DependencyFailure as exc:
self.breaker.failure()
raise DependencyFailure("invalid_safety_response") from exc
if body["task_id"] != location_task_id:
self.breaker.failure()
raise DependencyFailure("invalid_safety_response")
try:
if int(retry_after) <= 0 or body["poll_after_ms"] <= 0:
raise ValueError
datetime_value = body["expires_at"].replace("Z", "+00:00")
datetime.fromisoformat(datetime_value)
except (AttributeError, TypeError, ValueError) as exc:
self.breaker.failure()
raise DependencyFailure("invalid_safety_response") from exc
body["_location"] = location
body["_retry_after"] = retry_after
elif not body.get("rule_id") or (
status == 403 and body.get("reason_code") != "message_blocked"
):
self.breaker.failure()
raise DependencyFailure("invalid_safety_response")
self.breaker.success()
body["_status"] = response.status_code
return body
async def ready(self) -> bool:
try:
response = await self.http.get(
f"{str(self.settings.message_safety_url).rstrip('/')}/health/ready",
timeout=2,
)
return response.status_code == 200
except httpx.HTTPError:
return False
class OpenLinesClient:
def __init__(self, settings: Settings, http: httpx.AsyncClient) -> None:
self.settings = settings
self.http = http
self.breaker = CircuitBreaker(
settings.bitrix_local_app_circuit_failure_threshold,
settings.bitrix_local_app_circuit_open_sec,
)
async def send(
self, message_id: uuid.UUID, payload: dict[str, Any], request_id: str
) -> dict[str, Any]:
if not self.breaker.allow():
raise DependencyFailure()
try:
response = await self.http.post(
f"{str(self.settings.bitrix_local_app_base_url).rstrip('/')}"
"/internal/openlines/v1/messages",
json=payload,
headers={
"Authorization": "Bearer "
+ self.settings.bitrix_local_app_internal_token.get_secret_value(),
"Idempotency-Key": str(message_id),
"X-Request-ID": request_id,
},
timeout=self.settings.bitrix_local_app_http_timeout_sec,
)
response.raise_for_status()
except httpx.TimeoutException as exc:
self.breaker.failure()
raise DependencyFailure(timeout=True) from exc
except httpx.HTTPError as exc:
self.breaker.failure()
raise DependencyFailure() from exc
self.breaker.success()
return response.json()
async def ready(self) -> bool:
try:
response = await self.http.get(
f"{str(self.settings.bitrix_local_app_base_url).rstrip('/')}"
"/internal/openlines/v1/status",
headers={
"Authorization": "Bearer "
+ self.settings.bitrix_local_app_internal_token.get_secret_value()
},
timeout=2,
)
return response.is_success
except httpx.HTTPError:
return False
async def fresh_openlines_payload(
payload: dict[str, Any], s3: "S3Client"
) -> dict[str, Any]:
result = json.loads(json.dumps(payload, default=str))
for file in result.get("message", {}).get("files", []):
bucket = file.pop("_storage_bucket")
key = file.pop("_object_key")
file["download_url"] = await s3.presign_get(bucket, key)
return result
class S3Client:
def __init__(self, settings: Settings) -> None:
self.settings = settings
self.client = boto3.client(
"s3",
endpoint_url=str(settings.selectel_s3_endpoint_url),
aws_access_key_id=settings.selectel_s3_access_key.get_secret_value(),
aws_secret_access_key=settings.selectel_s3_secret_key.get_secret_value(),
config=Config(
signature_version="s3v4",
connect_timeout=3,
read_timeout=10,
retries={"max_attempts": 2},
s3={"addressing_style": "virtual"},
),
)
async def ready(self) -> bool:
try:
for bucket in (
self.settings.selectel_s3_bucket_quarantine,
self.settings.selectel_s3_bucket_attachments,
self.settings.selectel_s3_bucket_documents,
):
await asyncio.to_thread(self.client.head_bucket, Bucket=bucket)
return True
except Exception:
return False
async def presign_put(self, key: str, mime: str, ttl: int) -> str:
return await asyncio.to_thread(
self.client.generate_presigned_url,
"put_object",
Params={
"Bucket": self.settings.selectel_s3_bucket_quarantine,
"Key": key,
"ContentType": mime,
},
ExpiresIn=ttl,
)
async def presign_get(self, bucket: str, key: str, ttl: int = 300) -> str:
return await asyncio.to_thread(
self.client.generate_presigned_url,
"get_object",
Params={"Bucket": bucket, "Key": key},
ExpiresIn=ttl,
)
async def head(self, bucket: str, key: str) -> dict[str, Any]:
return await asyncio.to_thread(self.client.head_object, Bucket=bucket, Key=key)
async def promote(
self,
source_key: str,
destination_key: str,
*,
version_id: str,
etag: str,
) -> None:
await asyncio.to_thread(
self.client.copy_object,
Bucket=self.settings.selectel_s3_bucket_attachments,
Key=destination_key,
CopySource={
"Bucket": self.settings.selectel_s3_bucket_quarantine,
"Key": source_key,
"VersionId": version_id,
},
CopySourceIfMatch=etag,
)
# Keep the immutable source version until quarantine lifecycle expiry.
# A crash after copy but before the DB checkpoint can then safely retry
# the same conditional copy without losing its source.
async def delete_quarantine(self, key: str) -> None:
await asyncio.to_thread(
self.client.delete_object,
Bucket=self.settings.selectel_s3_bucket_quarantine,
Key=key,
)
async def delete(self, bucket: str, key: str) -> None:
await asyncio.to_thread(self.client.delete_object, Bucket=bucket, Key=key)
async def upload_inbound(
self,
http: httpx.AsyncClient,
download_url: str,
destination_key: str,
mime_type: str,
max_bytes: int,
) -> tuple[int, str]:
parsed = urlparse(download_url)
if parsed.scheme != "https" or not parsed.hostname:
raise DependencyFailure("unsafe_inbound_url")
try:
addresses = await asyncio.to_thread(
socket.getaddrinfo, parsed.hostname, parsed.port or 443, type=socket.SOCK_STREAM
)
except OSError as exc:
raise DependencyFailure("unsafe_inbound_url") from exc
if any(
ipaddress.ip_address(address[4][0]).is_private
or ipaddress.ip_address(address[4][0]).is_loopback
or ipaddress.ip_address(address[4][0]).is_link_local
or ipaddress.ip_address(address[4][0]).is_reserved
for address in addresses
):
raise DependencyFailure("unsafe_inbound_url")
data = bytearray()
try:
async with http.stream(
"GET", download_url, timeout=10, follow_redirects=False
) as response:
response.raise_for_status()
if response.headers.get("content-type", "").split(";")[0] != mime_type:
raise DependencyFailure("inbound_mime_mismatch")
async for chunk in response.aiter_bytes():
data.extend(chunk)
if len(data) > max_bytes:
raise DependencyFailure("inbound_file_too_large")
except httpx.HTTPError as exc:
raise DependencyFailure() from exc
await asyncio.to_thread(
self.client.put_object,
Bucket=self.settings.selectel_s3_bucket_attachments,
Key=destination_key,
Body=bytes(data),
ContentType=mime_type,
)
return len(data), hashlib.sha256(data).hexdigest()
VM1_app/codebase/backend/api-backend/app/logging_security.py
+47
View File
@@ -0,0 +1,47 @@
from __future__ import annotations
import re
from collections.abc import Mapping
from typing import Any
REDACTED = "[REDACTED]"
_SENSITIVE_KEY = re.compile(
r"(authorization|cookie|password|passwd|secret|token|api[_-]?key|"
r"database[_-]?url|redis[_-]?url|dsn|callback[_-]?url)",
re.IGNORECASE,
)
_URI_USERINFO = re.compile(r"(?P<scheme>[a-z][a-z0-9+.-]*://)[^/@\s]+@", re.IGNORECASE)
_QUERY_SECRET = re.compile(
r"(?P<prefix>[?&](?:token|access_token|api_key|key|secret|password)=)[^&#\s]+",
re.IGNORECASE,
)
_AUTH_VALUE = re.compile(r"\b(Bearer|Basic)\s+[A-Za-z0-9._~+/=-]+", re.IGNORECASE)
def sanitize_text(value: str) -> str:
value = _URI_USERINFO.sub(r"\g<scheme>[REDACTED]@", value)
value = _QUERY_SECRET.sub(r"\g<prefix>[REDACTED]", value)
return _AUTH_VALUE.sub(r"\1 [REDACTED]", value)
def sanitize_value(value: Any) -> Any:
if isinstance(value, str):
return sanitize_text(value)
if isinstance(value, Mapping):
return {
str(key): REDACTED if _SENSITIVE_KEY.search(str(key)) else sanitize_value(item)
for key, item in value.items()
}
if isinstance(value, list):
return [sanitize_value(item) for item in value]
if isinstance(value, tuple):
return tuple(sanitize_value(item) for item in value)
return value
def redact_event(
_logger: Any,
_method_name: str,
event_dict: dict[str, Any],
) -> dict[str, Any]:
return sanitize_value(event_dict)
VM1_app/codebase/backend/api-backend/app/main.py
+1174
View File
File diff suppressed because it is too large Load Diff
VM1_app/codebase/backend/api-backend/app/metrics.py
+21
View File
@@ -0,0 +1,21 @@
from opentelemetry import metrics
meter = metrics.get_meter("han.api")
HTTP_REQUESTS = meter.create_counter(
"han_http_requests_total",
description="Completed HTTP requests",
)
HTTP_DURATION = meter.create_histogram(
"han_http_request_duration_seconds",
unit="s",
description="HTTP request duration",
)
AUTH_BOOTSTRAP = meter.create_counter(
"han_auth_bootstrap_total",
description="Authentication bootstrap outcomes",
)
RATE_LIMIT_DECISIONS = meter.create_counter(
"han_rate_limit_decisions_total",
description="Rate-limit decisions",
)
VM1_app/codebase/backend/api-backend/app/notification_models.py
+303
View File
@@ -0,0 +1,303 @@
import secrets
import time
import uuid
from datetime import datetime
from decimal import Decimal
from typing import Any
from sqlalchemy import (
ARRAY,
BigInteger,
Boolean,
CheckConstraint,
DateTime,
ForeignKey,
Index,
Numeric,
SmallInteger,
String,
Text,
UniqueConstraint,
func,
)
from sqlalchemy.dialects.postgresql import JSONB, UUID
from sqlalchemy.orm import Mapped, mapped_column
from app.db import SCHEMA, Base, Common
def uuid7() -> uuid.UUID:
"""Generate an RFC 9562 UUIDv7 without relying on Python 3.14."""
timestamp_ms = int(time.time_ns() // 1_000_000) & ((1 << 48) - 1)
value = timestamp_ms << 80
value |= 0x7 << 76
value |= secrets.randbits(12) << 64
value |= 0b10 << 62
value |= secrets.randbits(62)
return uuid.UUID(int=value)
class NotificationCtaAction(Common, Base):
__tablename__ = "notification_cta_actions"
__table_args__ = (UniqueConstraint("code"), {"schema": SCHEMA})
code: Mapped[str] = mapped_column(String(32))
description: Mapped[str] = mapped_column(String(255))
requires_auth: Mapped[bool] = mapped_column(Boolean, default=False)
required_instance_fields: Mapped[list[str]] = mapped_column(
ARRAY(String(64)), default=list, server_default="{}"
)
class NotificationButton(Common, Base):
__tablename__ = "notification_buttons"
__table_args__ = (
UniqueConstraint("code"),
CheckConstraint("NOT applies_hidden_ttl OR sets_hidden"),
CheckConstraint("NOT submits_documents OR close_reason IS NOT NULL"),
{"schema": SCHEMA},
)
code: Mapped[str] = mapped_column(String(32))
label: Mapped[str] = mapped_column(String(64))
sets_hidden: Mapped[bool] = mapped_column(Boolean, default=False)
applies_hidden_ttl: Mapped[bool] = mapped_column(Boolean, default=False)
close_reason: Mapped[str | None] = mapped_column(String(32))
submits_documents: Mapped[bool] = mapped_column(Boolean, default=False)
class NotificationColorToken(Common, Base):
__tablename__ = "notification_color_tokens"
__table_args__ = (UniqueConstraint("code"), {"schema": SCHEMA})
code: Mapped[str] = mapped_column(String(32))
description: Mapped[str] = mapped_column(String(255))
sort_order: Mapped[int] = mapped_column(SmallInteger)
class NotificationType(Common, Base):
__tablename__ = "notification_types"
__table_args__ = (
UniqueConstraint("code"),
CheckConstraint("contour IN ('G','P')"),
CheckConstraint("contour <> 'G' OR countable = false"),
CheckConstraint("contour <> 'G' OR cta_action <> 'open_detail'"),
CheckConstraint(
"cta_action <> 'open_detail' OR (cta_sets_hidden = false AND cta_close_reason IS NULL)"
),
CheckConstraint(
"cta_action = 'open_detail' OR "
"(documents_allowed = false AND hide_on_document_download = false "
"AND required_detail_blocks = '{}')"
),
CheckConstraint("NOT hide_on_document_download OR documents_allowed"),
CheckConstraint("cta_action <> 'open_detail' OR button_primary_code IS NOT NULL"),
CheckConstraint(
"cta_action = 'open_detail' OR "
"(button_primary_code IS NULL AND button_secondary_code IS NULL)"
),
CheckConstraint("button_secondary_code IS NULL OR button_primary_code IS NOT NULL"),
CheckConstraint(
"button_secondary_code IS NULL OR button_secondary_code <> button_primary_code"
),
{"schema": SCHEMA},
)
code: Mapped[str] = mapped_column(String(32))
contour: Mapped[str] = mapped_column(String(1))
priority: Mapped[int] = mapped_column(SmallInteger)
countable: Mapped[bool] = mapped_column(Boolean)
label: Mapped[str] = mapped_column(String(64))
color_token: Mapped[str] = mapped_column(
ForeignKey(f"{SCHEMA}.notification_color_tokens.code", ondelete="RESTRICT")
)
icon_code: Mapped[str | None] = mapped_column(String(32))
cta_text: Mapped[str] = mapped_column(String(64))
cta_action: Mapped[str] = mapped_column(
ForeignKey(f"{SCHEMA}.notification_cta_actions.code", ondelete="RESTRICT")
)
cta_sets_hidden: Mapped[bool] = mapped_column(Boolean, default=False)
cta_close_reason: Mapped[str | None] = mapped_column(String(32))
button_primary_code: Mapped[str | None] = mapped_column(
ForeignKey(f"{SCHEMA}.notification_buttons.code", ondelete="RESTRICT")
)
button_secondary_code: Mapped[str | None] = mapped_column(
ForeignKey(f"{SCHEMA}.notification_buttons.code", ondelete="RESTRICT")
)
hidden_ttl_days: Mapped[int | None] = mapped_column(SmallInteger)
documents_allowed: Mapped[bool] = mapped_column(Boolean, default=False)
hide_on_document_download: Mapped[bool] = mapped_column(Boolean, default=False)
required_detail_blocks: Mapped[list[str]] = mapped_column(
ARRAY(String(64)), default=list, server_default="{}"
)
class NotificationSource(Common, Base):
__tablename__ = "notification_sources"
__table_args__ = (UniqueConstraint("code"), UniqueConstraint("token_hash"), {"schema": SCHEMA})
code: Mapped[str] = mapped_column(String(32))
description: Mapped[str | None] = mapped_column(Text)
token_hash: Mapped[str] = mapped_column(String(128))
token_rotated_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
class Notification(Common, Base):
__tablename__ = "notifications"
__table_args__ = (
UniqueConstraint("source", "external_id", name="uq_notifications_source_key"),
CheckConstraint("lifecycle_status IN ('active','closed')"),
CheckConstraint("visibility IN ('visible','hidden')"),
CheckConstraint(
"close_reason IS NULL OR close_reason IN "
"('user_done','docs_submitted','offer_accepted','paid','expired','cancelled')"
),
CheckConstraint(
"lifecycle_status <> 'closed' OR (close_reason IS NOT NULL AND closed_at IS NOT NULL)"
),
CheckConstraint("old_price IS NULL OR price IS NOT NULL"),
Index(
"ix_notifications_user_active",
"user_id",
"lifecycle_status",
"visibility",
"notification_datetime",
"id",
),
Index("ix_notifications_expire", "date_expired"),
{"schema": SCHEMA},
)
id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), primary_key=True, default=uuid7)
user_id: Mapped[uuid.UUID] = mapped_column(
ForeignKey(f"{SCHEMA}.user_identities.id", ondelete="RESTRICT")
)
notification_type: Mapped[str] = mapped_column(
ForeignKey(f"{SCHEMA}.notification_types.code", ondelete="RESTRICT")
)
source: Mapped[str] = mapped_column(
ForeignKey(f"{SCHEMA}.notification_sources.code", ondelete="RESTRICT")
)
external_id: Mapped[str] = mapped_column(String(128))
request_fingerprint: Mapped[str] = mapped_column(String(64))
notification_datetime: Mapped[datetime] = mapped_column(DateTime(timezone=True))
header: Mapped[str] = mapped_column(String(255))
text: Mapped[str | None] = mapped_column(String(1024))
priority_override: Mapped[int | None] = mapped_column(SmallInteger)
date_expired: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
price: Mapped[Decimal | None] = mapped_column(Numeric(12, 2))
old_price: Mapped[Decimal | None] = mapped_column(Numeric(12, 2))
payment_url: Mapped[str | None] = mapped_column(Text)
details: Mapped[dict[str, Any] | None] = mapped_column(JSONB)
details_schema_version: Mapped[int] = mapped_column(SmallInteger, default=1)
chat_message_text: Mapped[str | None] = mapped_column(String(1024))
lifecycle_status: Mapped[str] = mapped_column(String(16), default="active")
visibility: Mapped[str] = mapped_column(String(16), default="visible")
is_read: Mapped[bool] = mapped_column(Boolean, default=False)
close_reason: Mapped[str | None] = mapped_column(String(32))
closed_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
class GuestNotification(Common, Base):
__tablename__ = "guest_notifications"
__table_args__ = (
CheckConstraint("lifecycle_status IN ('active','closed')"),
CheckConstraint("old_price IS NULL OR price IS NOT NULL"),
CheckConstraint("instruction_url IS NULL OR instruction_url LIKE 'https://%'"),
Index("ix_guest_notifications_active", "lifecycle_status", "notification_datetime", "id"),
Index("ix_guest_notifications_expire", "date_expired"),
{"schema": SCHEMA},
)
id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), primary_key=True, default=uuid7)
notification_type: Mapped[str] = mapped_column(
ForeignKey(f"{SCHEMA}.notification_types.code", ondelete="RESTRICT")
)
notification_datetime: Mapped[datetime] = mapped_column(DateTime(timezone=True))
header: Mapped[str] = mapped_column(String(255))
text: Mapped[str | None] = mapped_column(String(1024))
priority_override: Mapped[int | None] = mapped_column(SmallInteger)
date_expired: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
price: Mapped[Decimal | None] = mapped_column(Numeric(12, 2))
old_price: Mapped[Decimal | None] = mapped_column(Numeric(12, 2))
instruction_url: Mapped[str | None] = mapped_column(Text)
chat_message_text: Mapped[str | None] = mapped_column(String(1024))
lifecycle_status: Mapped[str] = mapped_column(String(16), default="active")
closed_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
class NotificationDocument(Common, Base):
__tablename__ = "notification_documents"
__table_args__ = (
UniqueConstraint("notification_id", "document_id"),
{"schema": SCHEMA},
)
notification_id: Mapped[uuid.UUID] = mapped_column(
ForeignKey(f"{SCHEMA}.notifications.id", ondelete="RESTRICT")
)
document_id: Mapped[uuid.UUID] = mapped_column(
ForeignKey(f"{SCHEMA}.documents.id", ondelete="RESTRICT")
)
sort_order: Mapped[int] = mapped_column(SmallInteger, default=0)
download_url_issued_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
class ClientUploadDraft(Base):
__tablename__ = "client_upload_drafts"
__table_args__ = (
CheckConstraint("context_type IN ('notification')"),
CheckConstraint("size_bytes > 0"),
CheckConstraint("scan_status IN ('pending','clean','bypassed','infected','failed')"),
CheckConstraint("state IN ('draft','submitted','discarded')"),
Index("ix_client_upload_drafts_context", "user_id", "context_type", "context_id"),
Index("ix_client_upload_drafts_scan", "scan_status", "updated_at"),
Index("ix_client_upload_drafts_created", "created_at"),
{"schema": SCHEMA},
)
id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), primary_key=True, default=uuid7)
user_id: Mapped[uuid.UUID] = mapped_column(
ForeignKey(f"{SCHEMA}.user_identities.id", ondelete="RESTRICT")
)
context_type: Mapped[str] = mapped_column(String(32))
context_id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True))
original_file_name: Mapped[str] = mapped_column(String(255))
safe_file_name: Mapped[str] = mapped_column(String(255))
mime_type: Mapped[str] = mapped_column(String(128))
size_bytes: Mapped[int] = mapped_column(BigInteger)
checksum_sha256: Mapped[str | None] = mapped_column(String(64))
scan_status: Mapped[str] = mapped_column(String(16), default="pending")
storage_bucket: Mapped[str] = mapped_column(String(255))
object_key: Mapped[str] = mapped_column(String(1024))
quarantine_object_key: Mapped[str | None] = mapped_column(String(1024))
quarantine_version_id: Mapped[str | None] = mapped_column(String(1024))
quarantine_etag: Mapped[str | None] = mapped_column(String(1024))
safety_processing_mode: Mapped[str | None] = mapped_column(String(16))
safety_config_version: Mapped[int | None] = mapped_column(BigInteger)
safety_rules_version: Mapped[str | None] = mapped_column(String(128))
upload_expires_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
completed_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
state: Mapped[str] = mapped_column(String(16), default="draft")
submission_id: Mapped[uuid.UUID | None] = mapped_column(UUID(as_uuid=True))
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
updated_at: Mapped[datetime] = mapped_column(
DateTime(timezone=True), server_default=func.now(), onupdate=func.now()
)
class ClientDocument(Common, Base):
__tablename__ = "client_documents"
__table_args__ = (
UniqueConstraint("storage_bucket", "object_key"),
UniqueConstraint("source_draft_id"),
Index("ix_client_documents_context", "context_type", "context_id"),
Index("ix_client_documents_user_submitted", "user_id", "submitted_at"),
{"schema": SCHEMA},
)
user_id: Mapped[uuid.UUID] = mapped_column(
ForeignKey(f"{SCHEMA}.user_identities.id", ondelete="RESTRICT")
)
context_type: Mapped[str] = mapped_column(String(32))
context_id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True))
submission_id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True))
source_draft_id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True))
original_file_name: Mapped[str] = mapped_column(String(255))
safe_file_name: Mapped[str] = mapped_column(String(255))
mime_type: Mapped[str] = mapped_column(String(128))
size_bytes: Mapped[int] = mapped_column(BigInteger)
checksum_sha256: Mapped[str] = mapped_column(String(64))
storage_bucket: Mapped[str] = mapped_column(String(255))
object_key: Mapped[str] = mapped_column(String(1024))
submitted_at: Mapped[datetime] = mapped_column(DateTime(timezone=True))
VM1_app/codebase/backend/api-backend/app/notification_routes.py
+505
View File
@@ -0,0 +1,505 @@
import hashlib
import json
import uuid
from typing import Annotated
from fastapi import APIRouter, Depends, Header, Query, Request, Response
from fastapi.responses import JSONResponse
from sqlalchemy.ext.asyncio import AsyncSession
from app.auth import AuthError
from app.db import UserIdentity
from app.notification_models import NotificationSource
from app.notification_schemas import (
NotificationCancelRequest,
NotificationCreateRequest,
UploadCompleteRequest,
UploadInitRequest,
)
from app.notification_service import (
apply_read_or_hide,
authenticate_source,
cancel_notification,
catalog,
complete_upload,
create_notification,
discard_upload,
document_download,
init_upload,
invoke_cta_state,
list_notifications,
list_uploads,
notification_dto,
owned_notification,
press_button,
public_notifications,
unread_count,
)
from app.schemas import TextMessageRequest
from app.services import (
AuditContext,
DomainError,
SettingsSnapshot,
create_dialog,
load_settings,
resolve_user,
send_message,
)
router = APIRouter()
async def db_session(request: Request):
async for value in request.app.state.db.session():
yield value
Session = Annotated[AsyncSession, Depends(db_session)]
async def user_dependency(
request: Request,
db: Session,
authorization: Annotated[str | None, Header()] = None,
) -> UserIdentity:
if not authorization or not authorization.startswith("Bearer "):
raise AuthError()
principal = await request.app.state.jwks.validate(authorization.removeprefix("Bearer ").strip())
return await resolve_user(db, principal)
User = Annotated[UserIdentity, Depends(user_dependency)]
async def snapshot_dependency(db: Session) -> SettingsSnapshot:
return await load_settings(db)
Snapshot = Annotated[SettingsSnapshot, Depends(snapshot_dependency)]
async def source_dependency(
db: Session, authorization: Annotated[str | None, Header()] = None
) -> NotificationSource:
return await authenticate_source(db, authorization)
Source = Annotated[NotificationSource, Depends(source_dependency)]
def context(request: Request, ux_session: str | None = None) -> AuditContext:
try:
ux_id = uuid.UUID(ux_session) if ux_session else None
except ValueError:
raise DomainError("validation_error", 400, "X-Ux-Session-Id must be UUID") from None
return AuditContext(
request_id=request.state.request_id,
trace_id=request.state.trace_id,
ux_session_id=ux_id,
user_agent_hash=request.state.user_agent_hash,
client_ip=None,
)
async def rate_limit(
request: Request,
identity: str,
route: str,
limit: tuple[int, int],
*,
fail_closed: bool,
) -> None:
key = request.app.state.rate_limiter.key("notification", identity, route, limit[1])
try:
retry_after = await request.app.state.rate_limiter.consume(key, *limit)
except Exception:
if fail_closed:
raise DomainError(
"dependency_unavailable", 503, "Rate limit service is unavailable"
) from None
return
if retry_after:
raise DomainError(
"rate_limit_exceeded",
429,
"Rate limit exceeded",
{"retry_after": retry_after},
)
@router.get("/api/v1/public/notifications", tags=["notifications"])
async def public_list(request: Request, db: Session, settings: Snapshot):
await rate_limit(
request,
request.client.host if request.client else "unknown",
"public",
settings.limit("rate_limit.notifications_public.per_ip"),
fail_closed=False,
)
return {
"items": await public_notifications(db, settings.integer("notification.home.max_items"))
}
@router.get("/api/v1/public/notification-types", tags=["notifications"])
async def type_catalog(
request: Request,
response: Response,
db: Session,
settings: Snapshot,
if_none_match: Annotated[str | None, Header()] = None,
):
await rate_limit(
request,
request.client.host if request.client else "unknown",
"public",
settings.limit("rate_limit.notifications_public.per_ip"),
fail_closed=False,
)
items = await catalog(db)
digest = hashlib.sha256(
json.dumps(items, default=str, sort_keys=True, separators=(",", ":")).encode()
).hexdigest()
etag = f'"{digest}"'
if if_none_match == etag:
return Response(status_code=304, headers={"ETag": etag})
response.headers["ETag"] = etag
response.headers["Cache-Control"] = "public, max-age=3600"
return {"items": items}
@router.get("/api/v1/notifications", tags=["notifications"])
async def personal_list(
request: Request,
db: Session,
user: User,
settings: Snapshot,
place: str = Query(pattern="^(home|center)$"),
):
await rate_limit(
request,
str(user.id),
"read",
settings.limit("rate_limit.notifications_read.per_user"),
fail_closed=False,
)
limit = settings.integer(
"notification.home.max_items" if place == "home" else "notification.center.max_items"
)
return {"items": await list_notifications(db, user.id, place, limit)}
@router.get("/api/v1/notifications/counter", tags=["notifications"])
async def counter(request: Request, db: Session, user: User, settings: Snapshot):
await rate_limit(
request,
str(user.id),
"read",
settings.limit("rate_limit.notifications_read.per_user"),
fail_closed=False,
)
return {
"unread_count": await unread_count(
db, user.id, settings.integer("notification.center.max_items")
)
}
@router.get("/api/v1/notifications/{notification_id}", tags=["notifications"])
async def detail(
notification_id: uuid.UUID,
request: Request,
db: Session,
user: User,
settings: Snapshot,
):
await rate_limit(
request,
str(user.id),
"read",
settings.limit("rate_limit.notifications_read.per_user"),
fail_closed=False,
)
item, kind = await owned_notification(db, user.id, notification_id)
if kind.cta_action != "open_detail":
raise DomainError("not_found", 404, "Resource was not found")
return await notification_dto(db, item, kind)
@router.post("/api/v1/notifications/{notification_id}/read", tags=["notifications"])
async def mark_read(
notification_id: uuid.UUID,
request: Request,
db: Session,
user: User,
settings: Snapshot,
x_ux_session_id: Annotated[str | None, Header(alias="X-Ux-Session-Id")] = None,
):
await rate_limit(
request,
str(user.id),
"action",
settings.limit("rate_limit.notifications_action.per_user"),
fail_closed=False,
)
return await apply_read_or_hide(
db,
user.id,
notification_id,
"read",
settings,
request.app.state.realtime,
context(request, x_ux_session_id),
)
@router.post("/api/v1/notifications/{notification_id}/hide", tags=["notifications"])
async def hide(
notification_id: uuid.UUID,
request: Request,
db: Session,
user: User,
settings: Snapshot,
x_ux_session_id: Annotated[str | None, Header(alias="X-Ux-Session-Id")] = None,
):
await rate_limit(
request,
str(user.id),
"action",
settings.limit("rate_limit.notifications_action.per_user"),
fail_closed=False,
)
return await apply_read_or_hide(
db,
user.id,
notification_id,
"hide",
settings,
request.app.state.realtime,
context(request, x_ux_session_id),
)
@router.post(
"/api/v1/notifications/{notification_id}/buttons/{button_code}",
tags=["notifications"],
)
async def button(
notification_id: uuid.UUID,
button_code: str,
request: Request,
db: Session,
user: User,
settings: Snapshot,
x_ux_session_id: Annotated[str | None, Header(alias="X-Ux-Session-Id")] = None,
):
await rate_limit(
request,
str(user.id),
"action",
settings.limit("rate_limit.notifications_action.per_user"),
fail_closed=False,
)
return await press_button(
db,
user.id,
notification_id,
button_code,
settings,
request.app.state.realtime,
context(request, x_ux_session_id),
)
@router.post("/api/v1/notifications/{notification_id}/cta", tags=["notifications"])
async def cta(
notification_id: uuid.UUID,
request: Request,
db: Session,
user: User,
settings: Snapshot,
x_ux_session_id: Annotated[str | None, Header(alias="X-Ux-Session-Id")] = None,
):
await rate_limit(
request,
str(user.id),
"action",
settings.limit("rate_limit.notifications_action.per_user"),
fail_closed=False,
)
audit_context = context(request, x_ux_session_id)
item, kind = await owned_notification(db, user.id, notification_id, action=True)
chat_result = None
if kind.cta_action == "send_chat_message":
dialog, _status = await create_dialog(
db, user, audit_context, f"notification-dialog:{item.id}"
)
chat_result = await send_message(
db,
user,
uuid.UUID(str(dialog["dialog_id"])),
TextMessageRequest(content_kind="text", text=item.chat_message_text or ""),
f"notification-message:{item.id}",
audit_context,
settings,
request.app.state.settings,
request.app.state.safety,
request.app.state.openlines,
request.app.state.s3,
request.app.state.realtime,
)
_item, _kind, result = await invoke_cta_state(
db,
user.id,
notification_id,
settings,
request.app.state.realtime,
audit_context,
chat_result=chat_result,
)
return result
@router.get(
"/api/v1/notifications/{notification_id}/documents/{document_id}/download-url",
tags=["notifications"],
)
async def download(
notification_id: uuid.UUID,
document_id: uuid.UUID,
request: Request,
db: Session,
user: User,
settings: Snapshot,
x_ux_session_id: Annotated[str | None, Header(alias="X-Ux-Session-Id")] = None,
):
await rate_limit(
request,
str(user.id),
"download",
settings.limit("rate_limit.download_url.per_user"),
fail_closed=True,
)
return await document_download(
db,
user.id,
notification_id,
document_id,
settings,
request.app.state.s3,
request.app.state.realtime,
context(request, x_ux_session_id),
)
@router.post("/api/v1/uploads/init", status_code=201, tags=["uploads"])
async def upload_init(
body: UploadInitRequest,
request: Request,
db: Session,
user: User,
settings: Snapshot,
):
await rate_limit(
request,
str(user.id),
"upload",
settings.limit("rate_limit.notification_upload.per_user"),
fail_closed=True,
)
return await init_upload(db, user.id, body, settings, request.app.state.s3)
@router.post("/api/v1/uploads/{draft_id}/complete", tags=["uploads"])
async def upload_complete(
draft_id: uuid.UUID,
body: UploadCompleteRequest,
request: Request,
db: Session,
user: User,
settings: Snapshot,
):
await rate_limit(
request,
str(user.id),
"upload",
settings.limit("rate_limit.notification_upload.per_user"),
fail_closed=True,
)
return await complete_upload(
db,
user.id,
draft_id,
body,
request.app.state.s3,
request.app.state.safety,
request.state.request_id,
)
@router.get("/api/v1/uploads", tags=["uploads"])
async def uploads(
request: Request,
db: Session,
user: User,
settings: Snapshot,
context_type: str,
context_id: uuid.UUID,
):
await rate_limit(
request,
str(user.id),
"upload",
settings.limit("rate_limit.notification_upload.per_user"),
fail_closed=True,
)
if context_type != "notification":
raise DomainError("validation_error", 400, "Unsupported upload context")
return {"items": await list_uploads(db, user.id, context_type, context_id)}
@router.delete("/api/v1/uploads/{draft_id}", status_code=204, tags=["uploads"])
async def upload_delete(
draft_id: uuid.UUID,
request: Request,
db: Session,
user: User,
settings: Snapshot,
):
await rate_limit(
request,
str(user.id),
"upload",
settings.limit("rate_limit.notification_upload.per_user"),
fail_closed=True,
)
await discard_upload(db, user.id, draft_id, request.app.state.s3)
return Response(status_code=204)
@router.post("/internal/notifications/v1/notifications", tags=["internal"])
async def internal_create(
body: NotificationCreateRequest,
request: Request,
db: Session,
source: Source,
):
result, status = await create_notification(
db,
body,
source,
request.app.state.s3,
request.app.state.realtime,
context(request),
)
return JSONResponse(json.loads(json.dumps(result, default=str)), status_code=status)
@router.post("/internal/notifications/v1/notifications/cancel", tags=["internal"])
async def internal_cancel(
body: NotificationCancelRequest,
request: Request,
db: Session,
source: Source,
):
return await cancel_notification(db, body, source, request.app.state.realtime, context(request))
VM1_app/codebase/backend/api-backend/app/notification_schemas.py
+72
View File
@@ -0,0 +1,72 @@
import uuid
from datetime import datetime
from decimal import Decimal
from typing import Literal
from pydantic import Field, HttpUrl, model_validator
from app.schemas import StrictModel
class TodoItem(StrictModel):
number: int
text: str = Field(min_length=1, max_length=1024)
class CompanyDocumentInput(StrictModel):
object_key: str = Field(min_length=1, max_length=1024)
title: str = Field(min_length=1, max_length=255)
mime_type: str = Field(min_length=1, max_length=128)
size_bytes: int = Field(gt=0)
checksum_sha256: str = Field(pattern=r"^[0-9a-f]{64}$")
class NotificationDetailsInput(StrictModel):
deadline: datetime | None = None
details_header: str | None = Field(default=None, max_length=255)
details_text: str | None = Field(default=None, max_length=4000)
todo_header: str | None = Field(default=None, max_length=255)
todo_plan: list[TodoItem] | None = Field(default=None, min_length=1)
send_documents: bool = False
documents: list[CompanyDocumentInput] | None = Field(default=None, min_length=1)
class NotificationCreateRequest(StrictModel):
user_id: uuid.UUID
notification_type: str = Field(min_length=1, max_length=32)
source: str = Field(min_length=1, max_length=32)
external_id: str = Field(min_length=1, max_length=128)
notification_datetime: datetime
header: str = Field(min_length=1, max_length=255)
text: str | None = Field(default=None, max_length=1024)
priority_override: int | None = None
date_expired: datetime | None = None
price: Decimal | None = Field(default=None, ge=0, max_digits=12, decimal_places=2)
old_price: Decimal | None = Field(default=None, ge=0, max_digits=12, decimal_places=2)
payment_url: HttpUrl | None = None
chat_message_text: str | None = Field(default=None, max_length=1024)
details: NotificationDetailsInput | None = None
@model_validator(mode="after")
def prices(self) -> "NotificationCreateRequest":
if self.old_price is not None and self.price is None:
raise ValueError("old_price requires price")
return self
class NotificationCancelRequest(StrictModel):
source: str = Field(min_length=1, max_length=32)
external_id: str = Field(min_length=1, max_length=128)
close_reason: Literal["cancelled", "paid"]
class UploadInitRequest(StrictModel):
context_type: Literal["notification"]
context_id: uuid.UUID
file_name: str = Field(min_length=1, max_length=255)
mime_type: str = Field(min_length=1, max_length=128)
size_bytes: int = Field(gt=0)
class UploadCompleteRequest(StrictModel):
checksum: str = Field(pattern=r"^sha256:[0-9a-f]{64}$")
VM1_app/codebase/backend/api-backend/app/notification_service.py
+1177
View File
File diff suppressed because it is too large Load Diff
VM1_app/codebase/backend/api-backend/app/otp_settings.py
+44
View File
@@ -0,0 +1,44 @@
from collections.abc import Mapping
OTP_SETTING_KEYS = {
"otp.phone.max_send_attempts_per_24h",
"otp.phone.min_seconds_between_attempts",
"otp.phone.max_verify_attempts",
"otp.phone.code_length",
"otp.phone.ttl_seconds",
"otp.phone.sms_order_timeout_ms",
}
def validate_otp_settings(values: Mapping[str, str]) -> None:
parsed: dict[str, int] = {}
for key in OTP_SETTING_KEYS:
raw = values.get(key)
if raw is None:
continue
try:
value = int(raw)
except (TypeError, ValueError) as error:
raise ValueError(f"{key}: integer value expected") from error
if str(value) != raw:
raise ValueError(f"{key}: canonical integer value expected")
parsed[key] = value
positive = OTP_SETTING_KEYS - {"otp.phone.min_seconds_between_attempts"}
for key in positive:
if key in parsed and parsed[key] <= 0:
raise ValueError(f"{key}: value must be positive")
if parsed.get("otp.phone.min_seconds_between_attempts", 0) < 0:
raise ValueError("otp.phone.min_seconds_between_attempts: value must be non-negative")
code_length = parsed.get("otp.phone.code_length")
if code_length is not None and not 4 <= code_length <= 10:
raise ValueError("otp.phone.code_length: value must be between 4 and 10")
ttl_seconds = parsed.get("otp.phone.ttl_seconds")
if ttl_seconds is not None and (
not 60 <= ttl_seconds <= 900 or ttl_seconds % 60 != 0
):
raise ValueError(
"otp.phone.ttl_seconds: value must be between 60 and 900 and divisible by 60"
)
VM1_app/codebase/backend/api-backend/app/postgres.py
+25
View File
@@ -0,0 +1,25 @@
from __future__ import annotations
from typing import Any
import asyncpg
from sqlalchemy.ext.asyncio import AsyncEngine, create_async_engine
def asyncpg_dsn(url: str) -> str:
if url.startswith("postgresql+asyncpg://"):
return url.replace("postgresql+asyncpg://", "postgresql://", 1)
return url
def create_postgres_engine(url: str, **engine_options: Any) -> AsyncEngine:
dsn = asyncpg_dsn(url)
async def connect():
return await asyncpg.connect(dsn=dsn)
return create_async_engine(
"postgresql+asyncpg://",
async_creator=connect,
**engine_options,
)
VM1_app/codebase/backend/api-backend/app/realtime.py
+98
View File
@@ -0,0 +1,98 @@
import asyncio
import json
import uuid
from collections.abc import AsyncIterator
from contextlib import suppress
from typing import Any
from redis.asyncio import Redis
DIALOG_CHANNEL_PREFIX = "han:rt:dialog:"
USER_CHANNEL_PREFIX = "han:rt:user:"
# Backward-compatible name used by existing chat integrations.
CHANNEL_PREFIX = DIALOG_CHANNEL_PREFIX
class LocalFanout:
def __init__(self) -> None:
self._queues: set[asyncio.Queue[dict[str, Any]]] = set()
async def publish(self, event: dict[str, Any]) -> None:
for queue in tuple(self._queues):
with suppress(asyncio.QueueFull):
queue.put_nowait(event)
async def subscribe(self) -> AsyncIterator[dict[str, Any]]:
queue: asyncio.Queue[dict[str, Any]] = asyncio.Queue(maxsize=256)
self._queues.add(queue)
try:
while True:
yield await queue.get()
finally:
self._queues.discard(queue)
class RealtimeFanout:
def __init__(self, redis: Redis, local: LocalFanout | None = None) -> None:
self.redis = redis
self.local = local or LocalFanout()
async def publish(self, event: dict[str, Any]) -> None:
event = {"event_id": str(uuid.uuid4()), **event}
channel = DIALOG_CHANNEL_PREFIX + str(event["dialog_id"])
try:
await self.redis.publish(channel, json.dumps(event, default=str, separators=(",", ":")))
except Exception:
await self.local.publish(event)
async def publish_user(self, user_id: uuid.UUID, event: dict[str, Any]) -> None:
event = {"event_id": str(uuid.uuid4()), "_user_id": str(user_id), **event}
channel = USER_CHANNEL_PREFIX + str(user_id)
try:
await self.redis.publish(channel, json.dumps(event, default=str, separators=(",", ":")))
except Exception:
await self.local.publish(event)
async def events(
self,
dialog_ids: set[uuid.UUID],
user_id: uuid.UUID | None = None,
notifications: bool = False,
) -> AsyncIterator[dict[str, Any]]:
channels = [DIALOG_CHANNEL_PREFIX + str(dialog_id) for dialog_id in dialog_ids]
if notifications and user_id is not None:
channels.append(USER_CHANNEL_PREFIX + str(user_id))
if not channels:
await asyncio.Event().wait()
return
pubsub = self.redis.pubsub()
try:
await pubsub.subscribe(*channels)
except Exception:
await pubsub.aclose()
async for event in self.local.subscribe():
dialog_match = event.get("dialog_id") and uuid.UUID(
str(event["dialog_id"])
) in dialog_ids
user_match = (
notifications
and user_id is not None
and event.get("_user_id") == str(user_id)
)
if dialog_match or user_match:
payload = dict(event)
payload.pop("_user_id", None)
yield payload
return
try:
while True:
message = await pubsub.get_message(ignore_subscribe_messages=True, timeout=1)
if message:
payload = json.loads(message["data"])
payload.pop("_user_id", None)
yield payload
else:
await asyncio.sleep(0)
finally:
await pubsub.unsubscribe(*channels)
await pubsub.aclose()
VM1_app/codebase/backend/api-backend/app/schemas.py
+161
View File
@@ -0,0 +1,161 @@
import hashlib
import hmac
import json
import uuid
from base64 import urlsafe_b64decode, urlsafe_b64encode
from datetime import datetime
from enum import StrEnum
from typing import Annotated, Literal
from pydantic import BaseModel, ConfigDict, Field, HttpUrl, model_validator
from app.chat_settings import CHAT_MESSAGE_TRANSPORT_MAX_LENGTH
class StrictModel(BaseModel):
model_config = ConfigDict(extra="forbid")
class OtpSettingsResponse(StrictModel):
max_send_attempts_per_24h: int = Field(strict=True, gt=0)
min_seconds_between_attempts: int = Field(strict=True, ge=0)
max_verify_attempts: int = Field(strict=True, gt=0)
code_length: int = Field(strict=True, ge=4, le=10)
ttl_seconds: int = Field(strict=True, ge=60, le=900, multiple_of=60)
sms_order_timeout_ms: int = Field(strict=True, gt=0)
version: str = Field(min_length=1, max_length=64)
cache_ttl_seconds: int = Field(strict=True, gt=0)
class Device(StrictModel):
platform: Literal["ios", "android", "web"]
app_version: str = Field(min_length=1, max_length=64)
device_id: str | None = Field(default=None, max_length=255)
class ConsentChoice(StrictModel):
accepted: bool
version: str = Field(min_length=1, max_length=64)
class ConsentSet(StrictModel):
personal_data: ConsentChoice
user_agreement: ConsentChoice
marketing: ConsentChoice
class BootstrapRequest(StrictModel):
consents: ConsentSet
device: Device
class ConsentsRequest(StrictModel):
consents: ConsentSet
class SessionStartRequest(StrictModel):
start_reason: Literal["first_launch", "cold_start", "idle_timeout"]
device: Device
class TextMessageRequest(StrictModel):
content_kind: Literal["text"]
text: str = Field(min_length=1, max_length=CHAT_MESSAGE_TRANSPORT_MAX_LENGTH)
class FileMessageRequest(StrictModel):
content_kind: Literal["file"]
attachment_id: uuid.UUID
checksum: str = Field(pattern=r"^sha256:[0-9a-f]{64}$")
MessageRequest = Annotated[
TextMessageRequest | FileMessageRequest, Field(discriminator="content_kind")
]
class AttachmentInitRequest(StrictModel):
file_name: str = Field(min_length=1, max_length=255)
mime_type: str = Field(min_length=1, max_length=128)
size_bytes: int = Field(gt=0)
class AttachmentCompleteRequest(StrictModel):
checksum: str = Field(pattern=r"^sha256:[0-9a-f]{64}$")
class OpenLinesFile(StrictModel):
name: str = Field(min_length=1, max_length=255)
mime_type: str = Field(min_length=1, max_length=128)
size_bytes: int = Field(gt=0)
download_url: HttpUrl
class OpenLinesMessage(StrictModel):
text: str = Field(default="", max_length=4000)
files: list[OpenLinesFile] = Field(default_factory=list, max_length=1)
@model_validator(mode="after")
def non_empty(self) -> "OpenLinesMessage":
if not self.text.strip() and not self.files:
raise ValueError("message must contain text or file")
return self
class OpenLinesInbox(StrictModel):
event_id: str = Field(min_length=1, max_length=255)
event_type: Literal["message.new", "dialog.closed"]
external_chat_id: uuid.UUID
bitrix_message_id: str | None = Field(default=None, max_length=255)
occurred_at: datetime
message: OpenLinesMessage | None = None
@model_validator(mode="after")
def event_shape(self) -> "OpenLinesInbox":
if self.event_type == "message.new" and (
not self.bitrix_message_id or self.message is None
):
raise ValueError("message.new requires bitrix_message_id and message")
return self
class DialogStatus(StrEnum):
OPEN = "open"
WAITING_COMPANY = "waiting_for_company"
WAITING_CLIENT = "waiting_for_client"
CLOSED = "closed"
def canonical_fingerprint(
method: str, route: str, path_params: dict[str, str], body: object, user_id: uuid.UUID
) -> str:
value = {
"method": method.upper(),
"route": route,
"path": dict(sorted(path_params.items())),
"body": body,
"user_id": str(user_id),
}
return hashlib.sha256(
json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=False).encode()
).hexdigest()
def encode_cursor(data: dict[str, str], secret: bytes) -> str:
payload = json.dumps({"v": 1, **data}, sort_keys=True, separators=(",", ":")).encode()
signature = hmac.digest(secret, payload, "sha256")
return urlsafe_b64encode(payload + signature).decode().rstrip("=")
def decode_cursor(value: str, secret: bytes) -> dict[str, str]:
try:
raw = urlsafe_b64decode(value + "=" * (-len(value) % 4))
payload, signature = raw[:-32], raw[-32:]
if not hmac.compare_digest(signature, hmac.digest(secret, payload, "sha256")):
raise ValueError("invalid cursor")
decoded = json.loads(payload)
if decoded.pop("v") != 1:
raise ValueError("unsupported cursor")
return decoded
except (ValueError, KeyError, json.JSONDecodeError) as exc:
raise ValueError("invalid cursor") from exc
VM1_app/codebase/backend/api-backend/app/services.py
+1229
View File
File diff suppressed because it is too large Load Diff
VM1_app/codebase/backend/api-backend/app/settings.py
+95
View File
@@ -0,0 +1,95 @@
from functools import lru_cache
from typing import Literal
from pydantic import AnyHttpUrl, Field, SecretStr, model_validator
from pydantic_settings import BaseSettings, SettingsConfigDict
class Settings(BaseSettings):
model_config = SettingsConfigDict(
env_file=".env", env_file_encoding="utf-8", case_sensitive=True, extra="ignore"
)
app_env: str = Field(alias="APP_ENV")
api_port: int = Field(default=8000, alias="API_PORT")
log_level: str = Field(default="INFO", alias="LOG_LEVEL")
database_url: str = Field(alias="DATABASE_URL")
redis_url: str = Field(alias="REDIS_URL")
redis_realtime_url: str = Field(alias="REDIS_REALTIME_URL")
keycloak_public_url: AnyHttpUrl = Field(alias="KEYCLOAK_PUBLIC_URL")
keycloak_internal_url: AnyHttpUrl = Field(alias="KEYCLOAK_INTERNAL_URL")
keycloak_realm: str = Field(alias="KEYCLOAK_REALM")
keycloak_audience: str = Field(alias="KEYCLOAK_AUDIENCE")
jwks_cache_ttl_seconds: int = 300
jwks_stale_grace_seconds: int = 900
message_safety_url: AnyHttpUrl = Field(alias="MESSAGE_SAFETY_URL")
message_safety_service_token: SecretStr = Field(alias="MESSAGE_SAFETY_SERVICE_TOKEN")
message_safety_ca_file: str | None = Field(default=None, alias="MESSAGE_SAFETY_CA_FILE")
message_safety_api_prefix: Literal["/internal/safety/v2"] = Field(
default="/internal/safety/v2", alias="MESSAGE_SAFETY_API_PREFIX"
)
message_safety_post_timeout_sec: float = Field(
default=5, alias="MESSAGE_SAFETY_POST_TIMEOUT_SEC"
)
message_safety_task_poll_interval_sec: float = Field(
default=2, alias="MESSAGE_SAFETY_TASK_POLL_INTERVAL_SEC"
)
message_safety_task_poll_max_sec: float = Field(
default=300, alias="MESSAGE_SAFETY_TASK_POLL_MAX_SEC"
)
message_safety_circuit_failure_threshold: int = Field(
default=5, alias="MESSAGE_SAFETY_CIRCUIT_FAILURE_THRESHOLD"
)
message_safety_circuit_open_sec: int = Field(
default=30, alias="MESSAGE_SAFETY_CIRCUIT_OPEN_SEC"
)
bitrix_local_app_base_url: AnyHttpUrl = Field(alias="BITRIX_LOCAL_APP_BASE_URL")
bitrix_local_app_internal_token: SecretStr = Field(alias="BITRIX_LOCAL_APP_INTERNAL_TOKEN")
bitrix_api_inbox_token: SecretStr = Field(alias="BITRIX_API_INBOX_TOKEN")
bitrix_local_app_http_timeout_sec: float = Field(
default=20, alias="BITRIX_LOCAL_APP_HTTP_TIMEOUT_SEC"
)
bitrix_local_app_circuit_failure_threshold: int = Field(
default=5, alias="BITRIX_LOCAL_APP_CIRCUIT_FAILURE_THRESHOLD"
)
bitrix_local_app_circuit_open_sec: int = Field(
default=30, alias="BITRIX_LOCAL_APP_CIRCUIT_OPEN_SEC"
)
keycloak_settings_bridge_token: SecretStr = Field(alias="KEYCLOAK_SETTINGS_BRIDGE_TOKEN")
selectel_s3_endpoint_url: AnyHttpUrl = Field(alias="SELECTEL_S3_ENDPOINT_URL")
selectel_s3_bucket_documents: str = Field(alias="SELECTEL_S3_BUCKET_DOCUMENTS")
selectel_s3_bucket_attachments: str = Field(alias="SELECTEL_S3_BUCKET_ATTACHMENTS")
selectel_s3_bucket_quarantine: str = Field(alias="SELECTEL_S3_BUCKET_QUARANTINE")
selectel_s3_access_key: SecretStr = Field(alias="SELECTEL_S3_ACCESS_KEY")
selectel_s3_secret_key: SecretStr = Field(alias="SELECTEL_S3_SECRET_KEY")
otel_exporter_otlp_endpoint: str | None = Field(
default=None, alias="OTEL_EXPORTER_OTLP_ENDPOINT"
)
cursor_hmac_secret: SecretStr = Field(alias="CURSOR_HMAC_SECRET")
trusted_proxy_cidrs: str = Field(default="127.0.0.1/32", alias="TRUSTED_PROXY_CIDRS")
worker_poll_interval_sec: float = Field(default=2, alias="WORKER_POLL_INTERVAL_SEC")
notifications_token_producer_test: SecretStr | None = Field(
default=None, alias="NOTIFICATIONS_TOKEN_PRODUCER_TEST"
)
@model_validator(mode="after")
def require_safety_tls_in_deployed_environments(self) -> "Settings":
if self.app_env not in {"local", "test"}:
if str(self.message_safety_url).split(":", 1)[0] != "https":
raise ValueError("MESSAGE_SAFETY_URL must use HTTPS")
if not self.message_safety_ca_file:
raise ValueError("MESSAGE_SAFETY_CA_FILE is required")
return self
@property
def issuer(self) -> str:
return f"{str(self.keycloak_public_url).rstrip('/')}/realms/{self.keycloak_realm}"
@lru_cache
def get_settings() -> Settings:
return Settings()
VM1_app/codebase/backend/api-backend/app/telemetry.py
+111
View File
@@ -0,0 +1,111 @@
from __future__ import annotations
import os
from dataclasses import dataclass
from typing import Any
from fastapi import FastAPI
from opentelemetry import metrics, trace
from opentelemetry.exporter.otlp.proto.grpc.metric_exporter import OTLPMetricExporter
from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter
from opentelemetry.instrumentation.botocore import BotocoreInstrumentor
from opentelemetry.instrumentation.fastapi import FastAPIInstrumentor
from opentelemetry.instrumentation.httpx import HTTPXClientInstrumentor
from opentelemetry.instrumentation.redis import RedisInstrumentor
from opentelemetry.instrumentation.sqlalchemy import SQLAlchemyInstrumentor
from opentelemetry.propagate import set_global_textmap
from opentelemetry.sdk.metrics import MeterProvider
from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader
from opentelemetry.sdk.resources import Resource
from opentelemetry.sdk.trace import TracerProvider
from opentelemetry.sdk.trace.export import BatchSpanProcessor
from opentelemetry.sdk.trace.sampling import ALWAYS_ON
from opentelemetry.trace.propagation.tracecontext import TraceContextTextMapPropagator
@dataclass(slots=True)
class TelemetryRuntime:
tracer_provider: TracerProvider
meter_provider: MeterProvider
def shutdown(self) -> None:
self.meter_provider.shutdown()
self.tracer_provider.shutdown()
_runtime: TelemetryRuntime | None = None
def _resource(service_name: str) -> Resource:
return Resource.create(
{
"service.name": service_name,
"service.namespace": "han-chat",
"service.version": os.getenv("RELEASE_VERSION", "unknown"),
"deployment.environment": os.getenv("APP_ENV", "production-like"),
}
)
def init_telemetry(service_name: str | None = None) -> TelemetryRuntime | None:
global _runtime
if _runtime is not None:
return _runtime
endpoint = os.getenv("OTEL_EXPORTER_OTLP_ENDPOINT", "").strip()
if not endpoint:
return None
resource = _resource(service_name or os.getenv("OTEL_SERVICE_NAME", "api-backend"))
insecure = endpoint.startswith("http://")
set_global_textmap(TraceContextTextMapPropagator())
tracer_provider = TracerProvider(resource=resource, sampler=ALWAYS_ON)
tracer_provider.add_span_processor(
BatchSpanProcessor(
OTLPSpanExporter(endpoint=endpoint, insecure=insecure, timeout=3),
max_queue_size=2048,
schedule_delay_millis=5000,
max_export_batch_size=512,
export_timeout_millis=3000,
)
)
trace.set_tracer_provider(tracer_provider)
metric_reader = PeriodicExportingMetricReader(
OTLPMetricExporter(endpoint=endpoint, insecure=insecure, timeout=3),
export_interval_millis=30000,
export_timeout_millis=3000,
)
meter_provider = MeterProvider(resource=resource, metric_readers=[metric_reader])
metrics.set_meter_provider(meter_provider)
HTTPXClientInstrumentor().instrument()
SQLAlchemyInstrumentor().instrument(enable_commenter=False)
RedisInstrumentor().instrument()
BotocoreInstrumentor().instrument()
_runtime = TelemetryRuntime(tracer_provider, meter_provider)
return _runtime
def instrument_fastapi(app: FastAPI) -> None:
FastAPIInstrumentor.instrument_app(
app,
excluded_urls="/health/live,/health/ready,/nginx-health/live",
)
def add_trace_context(
_logger: Any,
_method_name: str,
event_dict: dict[str, Any],
) -> dict[str, Any]:
context = trace.get_current_span().get_span_context()
if context.is_valid:
event_dict["trace_id"] = format(context.trace_id, "032x")
event_dict["span_id"] = format(context.span_id, "016x")
return event_dict
def current_trace_id() -> str | None:
context = trace.get_current_span().get_span_context()
return format(context.trace_id, "032x") if context.is_valid else None
VM1_app/codebase/backend/api-backend/app/workers.py
+383
View File
@@ -0,0 +1,383 @@
import asyncio
import uuid
from datetime import UTC, datetime, timedelta
import httpx
import redis.asyncio as redis
import structlog
from sqlalchemy import delete, select
from app.db import (
Database,
DeliveryOutbox,
Dialog,
Message,
MessageAttachment,
SafetyTask,
UserIdentity,
)
from app.integrations import (
DependencyFailure,
OpenLinesClient,
S3Client,
SafetyClient,
fresh_openlines_payload,
)
from app.notification_models import ClientUploadDraft
from app.notification_service import expire_notifications
from app.realtime import RealtimeFanout
from app.services import load_settings, publish_dialog_status, publish_message_status
from app.settings import Settings, get_settings
log = structlog.get_logger()
async def delivery_once(
db: Database,
client: OpenLinesClient,
s3: S3Client,
fanout: RealtimeFanout,
settings: Settings,
worker_id: str,
batch_size: int = 20,
) -> int:
async with db.sessions() as session:
rows = (
(
await session.execute(
select(DeliveryOutbox)
.where(
DeliveryOutbox.status.in_(["pending", "retry"]),
DeliveryOutbox.next_attempt_at <= datetime.now(UTC),
)
.with_for_update(skip_locked=True)
.limit(batch_size)
)
)
.scalars()
.all()
)
ids = [row.id for row in rows]
for row in rows:
row.status = "processing"
row.locked_at = datetime.now(UTC)
row.locked_by = worker_id
await session.commit()
for row_id in ids:
async with db.sessions() as session:
row = await session.get(DeliveryOutbox, row_id, with_for_update=True)
if row is None:
continue
message = None
dialog = None
try:
payload = await fresh_openlines_payload(row.payload_json, s3)
await client.send(row.message_id, payload, f"worker-{worker_id}")
row.status = "delivered"
message = await session.get(Message, row.message_id)
if message:
message.delivery_status = "delivered"
dialog = await session.get(Dialog, message.dialog_id)
if dialog:
dialog.status = "waiting_for_company"
dialog.last_message_at = datetime.now(UTC)
except DependencyFailure:
row.attempt_count += 1
row.status = "dead_letter" if row.attempt_count >= 12 else "retry"
row.next_attempt_at = datetime.now(UTC) + timedelta(
seconds=min(3600, 2**row.attempt_count)
)
row.last_error_code = "dependency_unavailable"
row.locked_at = None
row.locked_by = None
await session.commit()
if message:
await publish_message_status(fanout, message, settings)
if dialog:
await publish_dialog_status(fanout, dialog)
return len(ids)
async def safety_once(
db: Database,
safety: SafetyClient,
s3: S3Client,
fanout: RealtimeFanout,
settings: Settings,
worker_id: str,
batch_size: int = 20,
) -> int:
async with db.sessions() as session:
rows = (
(
await session.execute(
select(SafetyTask)
.where(
SafetyTask.status.in_(["polling", "failed"]),
SafetyTask.next_poll_at <= datetime.now(UTC),
SafetyTask.expires_at > datetime.now(UTC),
)
.with_for_update(skip_locked=True)
.limit(batch_size)
)
)
.scalars()
.all()
)
ids = [row.id for row in rows]
for row in rows:
row.locked_at, row.locked_by = datetime.now(UTC), worker_id
await session.commit()
for task_id in ids:
async with db.sessions() as session:
task = await session.get(SafetyTask, task_id, with_for_update=True)
if task is None:
continue
message = None
try:
verdict = await safety.poll(task.poll_location, f"worker-{worker_id}")
message = await session.get(Message, task.message_id)
attachment = (
await session.get(MessageAttachment, task.attachment_id)
if task.attachment_id
else None
)
if verdict["_status"] == 200 and message:
message.safety_processing_mode = verdict["processing_mode"]
message.safety_config_version = verdict["config_version"]
message.safety_rules_version = verdict["rules_version"]
if attachment and attachment.quarantine_object_key:
destination = f"attachments/dialogs/{message.dialog_id}/{attachment.id}"
await s3.promote(
attachment.quarantine_object_key,
destination,
version_id=attachment.quarantine_version_id or "",
etag=attachment.quarantine_etag or "",
)
attachment.storage_bucket = s3.settings.selectel_s3_bucket_attachments
attachment.object_key = destination
attachment.quarantine_object_key = None
attachment.scan_status = (
"bypassed"
if verdict["processing_mode"] == "mock"
else "clean"
)
message.safety_status = "allowed"
task.status = "completed"
dialog = await session.get(Dialog, message.dialog_id)
user = (
await session.get(UserIdentity, dialog.user_id)
if dialog
else None
)
if dialog and user:
session.add(
DeliveryOutbox(
message_id=message.id,
external_chat_id=dialog.id,
payload_json={
"message_id": str(message.id),
"external_chat_id": str(dialog.id),
"occurred_at": message.occurred_at.isoformat(),
"user": {
"id": str(user.id),
"display_name": user.phone_number,
},
"message": {
"content_kind": message.content_kind,
"text": message.text,
"files": (
[{
"attachment_id": str(attachment.id),
"name": attachment.safe_file_name,
"mime_type": attachment.mime_type,
"size_bytes": attachment.size_bytes,
"_storage_bucket": attachment.storage_bucket,
"_object_key": attachment.object_key,
}]
if attachment
else []
),
},
},
next_attempt_at=datetime.now(UTC),
)
)
elif verdict["_status"] == 403 and message:
message.safety_processing_mode = verdict["processing_mode"]
message.safety_config_version = verdict["config_version"]
message.safety_rules_version = verdict["rules_version"]
message.text = ""
message.safety_status = "blocked"
message.delivery_status = "rejected"
if attachment and attachment.quarantine_object_key:
await s3.delete_quarantine(attachment.quarantine_object_key)
attachment.scan_status = "infected"
task.status = "completed"
else:
if verdict["_status"] == 202:
task.poll_location = verdict["_location"]
task.next_poll_at = datetime.now(UTC) + timedelta(seconds=2)
except DependencyFailure as exc:
task.attempt_count += 1
terminal = exc.terminal or (
exc.code == "task_not_found" and task.attempt_count >= 2
)
task.status = "terminal_failed" if terminal else "failed"
task.last_error_code = exc.code
if terminal and message:
message.delivery_status = "failed"
task.next_poll_at = datetime.now(UTC) + timedelta(
seconds=min(300, 2**task.attempt_count)
)
task.locked_at = None
task.locked_by = None
await session.commit()
if message:
await publish_message_status(fanout, message, settings)
return len(ids)
async def cleanup_once(db: Database, s3: S3Client, batch_size: int = 100) -> int:
async with db.sessions() as session:
rows = (
(
await session.execute(
select(MessageAttachment)
.where(
MessageAttachment.record_status == "A",
MessageAttachment.quarantine_object_key.is_not(None),
MessageAttachment.upload_expires_at < datetime.now(UTC),
MessageAttachment.message_id.is_(None),
)
.with_for_update(skip_locked=True)
.limit(batch_size)
)
)
.scalars()
.all()
)
for row in rows:
if row.quarantine_object_key:
await s3.delete_quarantine(row.quarantine_object_key)
row.record_status = "D"
row.status_changed_at = datetime.now(UTC)
row.status_change_reason = "expired_quarantine_cleanup"
await session.commit()
return len(rows)
async def loop(kind: str) -> None:
settings = get_settings()
db = Database(settings.database_url)
http = httpx.AsyncClient()
safety = SafetyClient(settings, http)
openlines = OpenLinesClient(settings, http)
s3 = S3Client(settings)
redis_rt = redis.from_url(settings.redis_realtime_url, decode_responses=True)
fanout = RealtimeFanout(redis_rt)
worker_id = f"{kind}-{uuid.uuid4()}"
try:
while True:
count = 0
if kind == "delivery":
count = await delivery_once(db, openlines, s3, fanout, settings, worker_id)
elif kind == "safety":
count = await safety_once(db, safety, s3, fanout, settings, worker_id)
else:
count = await cleanup_once(db, s3)
if not count:
await asyncio.sleep(settings.worker_poll_interval_sec)
finally:
await http.aclose()
await redis_rt.aclose()
await db.close()
async def notification_expire_loop() -> None:
settings = get_settings()
db = Database(settings.database_url)
try:
while True:
async with db.sessions() as session:
snapshot = await load_settings(session)
run_at = snapshot.values["notification.expire_job.run_at"]
hour, minute = (int(value) for value in run_at.split(":", 1))
now = datetime.now(UTC)
target = now.replace(hour=hour, minute=minute, second=0, microsecond=0)
if target <= now:
target += timedelta(days=1)
await asyncio.sleep((target - now).total_seconds())
async with db.sessions() as session:
personal, guest = await expire_notifications(session)
log.info(
"notification.expired_batch",
personal_count=personal,
guest_count=guest,
)
finally:
await db.close()
async def notification_draft_cleanup_once(db: Database, s3: S3Client) -> int:
async with db.sessions() as session:
snapshot = await load_settings(session)
cutoff = datetime.now(UTC) - timedelta(
days=snapshot.integer("notification.upload_draft.ttl_days")
)
rows = (
(
await session.execute(
select(ClientUploadDraft)
.where(ClientUploadDraft.created_at < cutoff)
.with_for_update(skip_locked=True)
.limit(100)
)
)
.scalars()
.all()
)
for row in rows:
if row.state != "submitted":
if row.quarantine_object_key:
await s3.delete_quarantine(row.quarantine_object_key)
elif row.object_key:
await s3.delete(row.storage_bucket, row.object_key)
await session.execute(
delete(ClientUploadDraft).where(ClientUploadDraft.id == row.id)
)
await session.commit()
return len(rows)
async def notification_draft_cleanup_loop() -> None:
settings = get_settings()
db = Database(settings.database_url)
s3 = S3Client(settings)
try:
while True:
count = await notification_draft_cleanup_once(db, s3)
if count < 100:
await asyncio.sleep(86400)
finally:
await db.close()
def delivery_main() -> None:
asyncio.run(loop("delivery"))
def safety_main() -> None:
asyncio.run(loop("safety"))
def cleanup_main() -> None:
asyncio.run(loop("cleanup"))
def notification_expire_main() -> None:
asyncio.run(notification_expire_loop())
def notification_draft_cleanup_main() -> None:
asyncio.run(notification_draft_cleanup_loop())
VM1_app/codebase/backend/api-backend/container-entrypoint.sh
+23
View File
@@ -0,0 +1,23 @@
#!/bin/sh
set -eu
for name in ${HAN_SECRET_VARS:-}; do
case "$name" in
""|[0-9]*|*[!A-Z0-9_]*)
echo "container secrets: invalid variable name" >&2
exit 64
;;
*) ;;
esac
eval "file=\${${name}_FILE:-}"
if [ -z "$file" ] || [ ! -r "$file" ]; then
echo "container secrets: missing file for $name" >&2
exit 66
fi
value=$(cat "$file")
export "$name=$value"
unset "${name}_FILE"
done
unset HAN_SECRET_VARS
exec "$@"
VM1_app/codebase/backend/api-backend/docker-compose.yml
+103
View File
@@ -0,0 +1,103 @@
services:
api-backend:
build: .
image: han-chat/api-backend:local
expose:
- "8000"
environment:
HAN_SECRET_VARS: >-
DATABASE_URL REDIS_URL REDIS_REALTIME_URL MESSAGE_SAFETY_SERVICE_TOKEN
BITRIX_LOCAL_APP_INTERNAL_TOKEN BITRIX_API_INBOX_TOKEN
KEYCLOAK_SETTINGS_BRIDGE_TOKEN SELECTEL_S3_ACCESS_KEY SELECTEL_S3_SECRET_KEY
CURSOR_HMAC_SECRET
DATABASE_URL_FILE: /run/secrets/api_database_url
REDIS_URL_FILE: /run/secrets/api_redis_url
REDIS_REALTIME_URL_FILE: /run/secrets/api_redis_realtime_url
MESSAGE_SAFETY_SERVICE_TOKEN_FILE: /run/secrets/message_safety_service_token
BITRIX_LOCAL_APP_INTERNAL_TOKEN_FILE: /run/secrets/bitrix_local_app_internal_token
BITRIX_API_INBOX_TOKEN_FILE: /run/secrets/bitrix_api_inbox_token
KEYCLOAK_SETTINGS_BRIDGE_TOKEN_FILE: /run/secrets/keycloak_settings_bridge_token
SELECTEL_S3_ACCESS_KEY_FILE: /run/secrets/selectel_s3_access_key
SELECTEL_S3_SECRET_KEY_FILE: /run/secrets/selectel_s3_secret_key
CURSOR_HMAC_SECRET_FILE: /run/secrets/cursor_hmac_secret
APP_ENV: ${APP_ENV}
API_PORT: ${API_PORT:-8000}
LOG_LEVEL: ${LOG_LEVEL:-INFO}
KEYCLOAK_PUBLIC_URL: ${KEYCLOAK_PUBLIC_URL}
KEYCLOAK_INTERNAL_URL: ${KEYCLOAK_INTERNAL_URL}
KEYCLOAK_REALM: ${KEYCLOAK_REALM}
KEYCLOAK_AUDIENCE: ${KEYCLOAK_AUDIENCE}
MESSAGE_SAFETY_URL: ${MESSAGE_SAFETY_URL}
MESSAGE_SAFETY_API_PREFIX: /internal/safety/v2
MESSAGE_SAFETY_CA_FILE: /run/config/message-safety-internal-ca.pem
MESSAGE_SAFETY_POST_TIMEOUT_SEC: ${MESSAGE_SAFETY_POST_TIMEOUT_SEC:-5}
MESSAGE_SAFETY_TASK_POLL_INTERVAL_SEC: ${MESSAGE_SAFETY_TASK_POLL_INTERVAL_SEC:-2}
MESSAGE_SAFETY_TASK_POLL_MAX_SEC: ${MESSAGE_SAFETY_TASK_POLL_MAX_SEC:-300}
MESSAGE_SAFETY_CIRCUIT_FAILURE_THRESHOLD: ${MESSAGE_SAFETY_CIRCUIT_FAILURE_THRESHOLD:-5}
MESSAGE_SAFETY_CIRCUIT_OPEN_SEC: ${MESSAGE_SAFETY_CIRCUIT_OPEN_SEC:-30}
BITRIX_LOCAL_APP_BASE_URL: ${BITRIX_LOCAL_APP_BASE_URL}
BITRIX_LOCAL_APP_HTTP_TIMEOUT_SEC: ${BITRIX_LOCAL_APP_HTTP_TIMEOUT_SEC:-20}
BITRIX_LOCAL_APP_CIRCUIT_FAILURE_THRESHOLD: ${BITRIX_LOCAL_APP_CIRCUIT_FAILURE_THRESHOLD:-5}
BITRIX_LOCAL_APP_CIRCUIT_OPEN_SEC: ${BITRIX_LOCAL_APP_CIRCUIT_OPEN_SEC:-30}
SELECTEL_S3_ENDPOINT_URL: ${SELECTEL_S3_ENDPOINT_URL}
SELECTEL_S3_BUCKET_DOCUMENTS: ${SELECTEL_S3_BUCKET_DOCUMENTS}
SELECTEL_S3_BUCKET_ATTACHMENTS: ${SELECTEL_S3_BUCKET_ATTACHMENTS}
SELECTEL_S3_BUCKET_QUARANTINE: ${SELECTEL_S3_BUCKET_QUARANTINE}
OTEL_EXPORTER_OTLP_ENDPOINT: ${OTEL_EXPORTER_OTLP_ENDPOINT}
secrets:
- api_database_url
- api_redis_url
- api_redis_realtime_url
- message_safety_service_token
- bitrix_local_app_internal_token
- bitrix_api_inbox_token
- keycloak_settings_bridge_token
- selectel_s3_access_key
- selectel_s3_secret_key
- cursor_hmac_secret
volumes:
- type: bind
source: ${MESSAGE_SAFETY_CA_HOST_PATH}
target: /run/config/message-safety-internal-ca.pem
read_only: true
healthcheck:
test:
- CMD
- python
- -c
- "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8000/health/live', timeout=2)"
interval: 10s
timeout: 3s
retries: 5
restart: unless-stopped
ulimits:
core: {soft: 0, hard: 0}
networks:
- backend
- observability
networks:
backend:
observability:
secrets:
api_database_url:
file: ${HAN_SECRETS_DIR:-/run/han-chat/secrets}/DATABASE_URL
api_redis_url:
file: ${HAN_SECRETS_DIR:-/run/han-chat/secrets}/REDIS_URL
api_redis_realtime_url:
file: ${HAN_SECRETS_DIR:-/run/han-chat/secrets}/REDIS_REALTIME_URL
message_safety_service_token:
file: ${HAN_SECRETS_DIR:-/run/han-chat/secrets}/MESSAGE_SAFETY_SERVICE_TOKEN
bitrix_local_app_internal_token:
file: ${HAN_SECRETS_DIR:-/run/han-chat/secrets}/BITRIX_LOCAL_APP_INTERNAL_TOKEN
bitrix_api_inbox_token:
file: ${HAN_SECRETS_DIR:-/run/han-chat/secrets}/BITRIX_API_INBOX_TOKEN
keycloak_settings_bridge_token:
file: ${HAN_SECRETS_DIR:-/run/han-chat/secrets}/KEYCLOAK_SETTINGS_BRIDGE_TOKEN
selectel_s3_access_key:
file: ${HAN_SECRETS_DIR:-/run/han-chat/secrets}/SELECTEL_S3_ACCESS_KEY
selectel_s3_secret_key:
file: ${HAN_SECRETS_DIR:-/run/han-chat/secrets}/SELECTEL_S3_SECRET_KEY
cursor_hmac_secret:
file: ${HAN_SECRETS_DIR:-/run/han-chat/secrets}/CURSOR_HMAC_SECRET
VM1_app/codebase/backend/api-backend/openapi.yaml
+588
View File
@@ -0,0 +1,588 @@
openapi: 3.1.0
info:
title: HAN Chat API
version: 1.0.0
servers:
- url: /
paths:
/health/live:
get:
operationId: healthLive
responses:
"200": {description: Process is live}
/health/ready:
get:
operationId: healthReady
responses:
"200": {description: Ready or partially degraded}
"503": {$ref: "#/components/responses/DependencyUnavailable"}
/api/v1/public/app-config:
get:
operationId: getPublicAppConfig
responses:
"200":
description: Public application configuration
content:
application/json:
schema:
type: object
required: [messages]
properties:
messages:
type: object
required: [max_text_length]
properties:
max_text_length: {type: integer, minimum: 1, maximum: 10000}
/api/v1/public/content:
get:
operationId: getPublicContent
parameters:
- {name: locale, in: query, schema: {type: string, default: ru}}
responses:
"200": {description: Active UI content}
/api/v1/auth/bootstrap:
post:
operationId: bootstrap
security: [{bearerAuth: []}]
requestBody:
required: true
content:
application/json:
schema: {$ref: "#/components/schemas/BootstrapRequest"}
responses:
"200": {description: Local user resolved}
"401": {$ref: "#/components/responses/Unauthorized"}
/api/v1/consents:
post:
operationId: recordConsents
security: [{bearerAuth: []}]
requestBody:
required: true
content:
application/json:
schema: {$ref: "#/components/schemas/ConsentsRequest"}
responses:
"201": {description: Immutable consent records saved}
/api/v1/analytics/session-start:
post:
operationId: startUxSession
security: [{bearerAuth: []}]
requestBody:
required: true
content:
application/json:
schema: {$ref: "#/components/schemas/SessionStartRequest"}
responses:
"201": {description: UX session created}
/api/v1/me:
get:
operationId: getCurrentProfile
security: [{bearerAuth: []}]
responses:
"200": {description: Block profile}
/api/v1/me/documents:
get:
operationId: listDocuments
security: [{bearerAuth: []}]
responses:
"200": {description: Cursor-paginated documents}
/api/v1/documents/{document_id}:
parameters:
- {$ref: "#/components/parameters/DocumentId"}
get:
operationId: getDocument
security: [{bearerAuth: []}]
responses:
"200": {description: Document metadata}
"404": {$ref: "#/components/responses/NotFound"}
/api/v1/documents/{document_id}/download-url:
parameters:
- {$ref: "#/components/parameters/DocumentId"}
get:
operationId: getDocumentDownloadUrl
security: [{bearerAuth: []}]
responses:
"200": {description: Short-lived presigned GET}
/api/v1/dialogs:
post:
operationId: createDialog
security: [{bearerAuth: []}]
parameters:
- {$ref: "#/components/parameters/IdempotencyKey"}
responses:
"200": {description: Existing active dialog}
"201": {description: New active dialog}
get:
operationId: listDialogs
security: [{bearerAuth: []}]
responses:
"200": {description: Cursor-paginated dialogs}
/api/v1/dialogs/{dialog_id}:
parameters:
- {$ref: "#/components/parameters/DialogId"}
get:
operationId: getDialog
security: [{bearerAuth: []}]
responses:
"200": {description: Dialog summary}
"404": {$ref: "#/components/responses/NotFound"}
/api/v1/dialogs/{dialog_id}/messages:
parameters:
- {$ref: "#/components/parameters/DialogId"}
get:
operationId: listMessages
security: [{bearerAuth: []}]
parameters:
- {name: after, in: query, schema: {type: string}}
- {name: limit, in: query, schema: {type: integer, minimum: 1, maximum: 100, default: 50}}
responses:
"200": {description: Message history or polling delta}
post:
operationId: sendMessage
security: [{bearerAuth: []}]
parameters:
- {$ref: "#/components/parameters/IdempotencyKey"}
requestBody:
required: true
content:
application/json:
schema:
oneOf:
- {$ref: "#/components/schemas/TextMessageRequest"}
- {$ref: "#/components/schemas/FileMessageRequest"}
discriminator: {propertyName: content_kind}
responses:
"201": {description: Safety-allowed and delivered message}
"422": {description: Message blocked, content redacted}
"503": {$ref: "#/components/responses/DependencyUnavailable"}
/api/v1/dialogs/{dialog_id}/attachments/init:
parameters:
- {$ref: "#/components/parameters/DialogId"}
post:
operationId: initAttachment
security: [{bearerAuth: []}]
requestBody:
required: true
content:
application/json:
schema: {$ref: "#/components/schemas/AttachmentInitRequest"}
responses:
"201": {description: Upload metadata and presigned PUT}
/api/v1/dialogs/{dialog_id}/attachments/{attachment_id}/complete:
parameters:
- {$ref: "#/components/parameters/DialogId"}
- {$ref: "#/components/parameters/AttachmentId"}
post:
operationId: completeAttachment
security: [{bearerAuth: []}]
requestBody:
required: true
content:
application/json:
schema: {$ref: "#/components/schemas/ChecksumRequest"}
responses:
"200": {description: Upload metadata verified}
/api/v1/dialogs/{dialog_id}/attachments/{attachment_id}/download-url:
parameters:
- {$ref: "#/components/parameters/DialogId"}
- {$ref: "#/components/parameters/AttachmentId"}
get:
operationId: getAttachmentDownloadUrl
security: [{bearerAuth: []}]
responses:
"200": {description: Audited short-lived presigned GET}
/api/v1/public/notifications:
get:
operationId: listGuestNotifications
responses:
"200":
description: Server-sorted active guest campaigns
content:
application/json:
schema: {$ref: "#/components/schemas/NotificationList"}
/api/v1/public/notification-types:
get:
operationId: getNotificationCatalog
parameters:
- {name: If-None-Match, in: header, schema: {type: string}}
responses:
"200": {description: Public catalog with ETag}
"304": {description: Catalog is unchanged}
/api/v1/notifications:
get:
operationId: listNotifications
security: [{bearerAuth: []}]
parameters:
- {name: place, in: query, required: true, schema: {type: string, enum: [home, center]}}
responses:
"200":
description: Server-sorted personal notifications
content:
application/json:
schema: {$ref: "#/components/schemas/NotificationList"}
/api/v1/notifications/counter:
get:
operationId: getNotificationCounter
security: [{bearerAuth: []}]
responses:
"200":
description: Unread count in the center window
content:
application/json:
schema: {$ref: "#/components/schemas/NotificationCounter"}
/api/v1/notifications/{notification_id}:
parameters:
- {$ref: "#/components/parameters/NotificationId"}
get:
operationId: getNotification
security: [{bearerAuth: []}]
responses:
"200": {description: Active notification detail}
"404": {$ref: "#/components/responses/NotFound"}
/api/v1/notifications/{notification_id}/read:
parameters:
- {$ref: "#/components/parameters/NotificationId"}
post:
operationId: readNotification
security: [{bearerAuth: []}]
responses:
"200": {description: Current notification state}
"409": {$ref: "#/components/responses/Conflict"}
/api/v1/notifications/{notification_id}/hide:
parameters:
- {$ref: "#/components/parameters/NotificationId"}
post:
operationId: hideNotification
security: [{bearerAuth: []}]
responses:
"200": {description: Current notification state}
"409": {$ref: "#/components/responses/Conflict"}
/api/v1/notifications/{notification_id}/buttons/{button_code}:
parameters:
- {$ref: "#/components/parameters/NotificationId"}
- {name: button_code, in: path, required: true, schema: {type: string}}
post:
operationId: pressNotificationButton
security: [{bearerAuth: []}]
responses:
"200": {description: Current notification state}
"409": {$ref: "#/components/responses/Conflict"}
"422": {$ref: "#/components/responses/Unprocessable"}
/api/v1/notifications/{notification_id}/cta:
parameters:
- {$ref: "#/components/parameters/NotificationId"}
post:
operationId: invokeNotificationCta
security: [{bearerAuth: []}]
responses:
"200": {description: CTA result and current state}
"409": {$ref: "#/components/responses/Conflict"}
/api/v1/notifications/{notification_id}/documents/{document_id}/download-url:
parameters:
- {$ref: "#/components/parameters/NotificationId"}
- {$ref: "#/components/parameters/DocumentId"}
get:
operationId: getNotificationDocumentDownloadUrl
security: [{bearerAuth: []}]
responses:
"200": {description: Audited short-lived presigned GET}
"404": {$ref: "#/components/responses/NotFound"}
/api/v1/uploads/init:
post:
operationId: initClientUpload
security: [{bearerAuth: []}]
requestBody:
required: true
content:
application/json:
schema: {$ref: "#/components/schemas/UploadInitRequest"}
responses:
"201": {description: Upload draft and presigned PUT}
/api/v1/uploads/{draft_id}/complete:
parameters:
- {$ref: "#/components/parameters/DraftId"}
post:
operationId: completeClientUpload
security: [{bearerAuth: []}]
requestBody:
required: true
content:
application/json:
schema: {$ref: "#/components/schemas/ChecksumRequest"}
responses:
"200": {description: Current scan state}
/api/v1/uploads:
get:
operationId: listClientUploads
security: [{bearerAuth: []}]
parameters:
- {name: context_type, in: query, required: true, schema: {type: string, enum: [notification]}}
- {name: context_id, in: query, required: true, schema: {type: string, format: uuid}}
responses:
"200": {description: Upload drafts for context}
/api/v1/uploads/{draft_id}:
parameters:
- {$ref: "#/components/parameters/DraftId"}
delete:
operationId: deleteClientUpload
security: [{bearerAuth: []}]
responses:
"204": {description: Draft discarded}
/internal/notifications/v1/notifications:
post:
operationId: createNotification
security: [{serviceBearer: []}]
requestBody:
required: true
content:
application/json:
schema: {$ref: "#/components/schemas/NotificationCreateRequest"}
responses:
"200": {description: Idempotent duplicate}
"201": {description: Notification created}
"409": {$ref: "#/components/responses/Conflict"}
/internal/notifications/v1/notifications/cancel:
post:
operationId: cancelNotification
security: [{serviceBearer: []}]
requestBody:
required: true
content:
application/json:
schema: {$ref: "#/components/schemas/NotificationCancelRequest"}
responses:
"200": {description: Notification closed or already closed}
"404": {$ref: "#/components/responses/NotFound"}
/internal/openlines/v1/inbox:
post:
operationId: applyOpenLinesInbox
security: [{serviceBearer: []}]
requestBody:
required: true
content:
application/json:
schema: {$ref: "#/components/schemas/OpenLinesInbox"}
responses:
"200": {description: Duplicate acknowledged}
"201": {description: Event applied}
/internal/settings/v1/otp:
get:
operationId: getOtpSettings
security: [{serviceBearer: []}]
responses:
"200":
description: Product OTP limits and cache metadata
content:
application/json:
schema: {$ref: "#/components/schemas/OtpSettingsResponse"}
"304": {description: Cached settings are still current}
"503": {$ref: "#/components/responses/DependencyUnavailable"}
components:
securitySchemes:
bearerAuth: {type: http, scheme: bearer, bearerFormat: JWT}
serviceBearer: {type: http, scheme: bearer}
parameters:
DialogId: {name: dialog_id, in: path, required: true, schema: {type: string, format: uuid}}
AttachmentId: {name: attachment_id, in: path, required: true, schema: {type: string, format: uuid}}
DocumentId: {name: document_id, in: path, required: true, schema: {type: string, format: uuid}}
NotificationId: {name: notification_id, in: path, required: true, schema: {type: string, format: uuid}}
DraftId: {name: draft_id, in: path, required: true, schema: {type: string, format: uuid}}
IdempotencyKey: {name: Idempotency-Key, in: header, required: true, schema: {type: string, minLength: 1, maxLength: 128}}
responses:
Unauthorized:
description: JWT is absent, invalid or expired
content: {application/json: {schema: {$ref: "#/components/schemas/ErrorEnvelope"}}}
NotFound:
description: Resource absent or owned by another user
content: {application/json: {schema: {$ref: "#/components/schemas/ErrorEnvelope"}}}
DependencyUnavailable:
description: Required dependency is unavailable
content: {application/json: {schema: {$ref: "#/components/schemas/ErrorEnvelope"}}}
Conflict:
description: Notification state or idempotency conflict
content: {application/json: {schema: {$ref: "#/components/schemas/ErrorEnvelope"}}}
Unprocessable:
description: Catalog action is not allowed
content: {application/json: {schema: {$ref: "#/components/schemas/ErrorEnvelope"}}}
schemas:
OtpSettingsResponse:
type: object
additionalProperties: false
required:
- max_send_attempts_per_24h
- min_seconds_between_attempts
- max_verify_attempts
- code_length
- ttl_seconds
- sms_order_timeout_ms
- version
- cache_ttl_seconds
properties:
max_send_attempts_per_24h: {type: integer, minimum: 1}
min_seconds_between_attempts: {type: integer, minimum: 0}
max_verify_attempts: {type: integer, minimum: 1}
code_length: {type: integer, minimum: 4, maximum: 10}
ttl_seconds: {type: integer, minimum: 60, maximum: 900, multipleOf: 60}
sms_order_timeout_ms: {type: integer, minimum: 1}
version: {type: string, minLength: 1, maxLength: 64}
cache_ttl_seconds: {type: integer, minimum: 1}
ErrorEnvelope:
type: object
required: [error]
properties:
error:
type: object
required: [code, message, request_id, details]
properties:
code: {type: string}
message: {type: string}
request_id: {type: string}
details: {type: object}
NotificationCounter:
type: object
additionalProperties: false
required: [unread_count]
properties:
unread_count: {type: integer, minimum: 0}
NotificationList:
type: object
additionalProperties: false
required: [items]
properties:
items:
type: array
items: {$ref: "#/components/schemas/Notification"}
Notification:
type: object
required: [id, notification_type, notification_datetime, header]
properties:
id: {type: string, format: uuid}
notification_type: {type: string}
notification_datetime: {type: string, format: date-time}
header: {type: string}
text: {type: [string, "null"]}
instruction_url: {type: [string, "null"], format: uri}
instruction_open_mode: {type: [string, "null"], enum: [new_tab, null]}
lifecycle_status: {type: string, enum: [active, closed]}
visibility: {type: string, enum: [visible, hidden]}
is_read: {type: boolean}
details: {type: [object, "null"]}
NotificationCreateRequest:
type: object
additionalProperties: false
required: [user_id, notification_type, source, external_id, notification_datetime, header]
properties:
user_id: {type: string, format: uuid}
notification_type: {type: string, maxLength: 32}
source: {type: string, maxLength: 32}
external_id: {type: string, maxLength: 128}
notification_datetime: {type: string, format: date-time}
header: {type: string, maxLength: 255}
text: {type: [string, "null"], maxLength: 1024}
priority_override: {type: [integer, "null"]}
date_expired: {type: [string, "null"], format: date-time}
price: {type: [number, "null"]}
old_price: {type: [number, "null"]}
payment_url: {type: [string, "null"], format: uri}
chat_message_text: {type: [string, "null"], maxLength: 1024}
details: {type: [object, "null"]}
NotificationCancelRequest:
type: object
additionalProperties: false
required: [source, external_id, close_reason]
properties:
source: {type: string}
external_id: {type: string}
close_reason: {type: string, enum: [cancelled, paid]}
UploadInitRequest:
type: object
additionalProperties: false
required: [context_type, context_id, file_name, mime_type, size_bytes]
properties:
context_type: {const: notification}
context_id: {type: string, format: uuid}
file_name: {type: string, maxLength: 255}
mime_type: {type: string, maxLength: 128}
size_bytes: {type: integer, minimum: 1}
ConsentChoice:
type: object
additionalProperties: false
required: [accepted, version]
properties:
accepted: {type: boolean}
version: {type: string, maxLength: 64}
ConsentSet:
type: object
additionalProperties: false
required: [personal_data, user_agreement, marketing]
properties:
personal_data: {$ref: "#/components/schemas/ConsentChoice"}
user_agreement: {$ref: "#/components/schemas/ConsentChoice"}
marketing: {$ref: "#/components/schemas/ConsentChoice"}
Device:
type: object
additionalProperties: false
required: [platform, app_version]
properties:
platform: {type: string, enum: [ios, android, web]}
app_version: {type: string, maxLength: 64}
device_id: {type: [string, "null"], maxLength: 255}
BootstrapRequest:
type: object
additionalProperties: false
required: [consents, device]
properties:
consents: {$ref: "#/components/schemas/ConsentSet"}
device: {$ref: "#/components/schemas/Device"}
ConsentsRequest:
type: object
additionalProperties: false
required: [consents]
properties: {consents: {$ref: "#/components/schemas/ConsentSet"}}
SessionStartRequest:
type: object
additionalProperties: false
required: [start_reason, device]
properties:
start_reason: {type: string, enum: [first_launch, cold_start, idle_timeout]}
device: {$ref: "#/components/schemas/Device"}
TextMessageRequest:
type: object
additionalProperties: false
required: [content_kind, text]
properties:
content_kind: {const: text}
text: {type: string, minLength: 1, maxLength: 10000}
FileMessageRequest:
type: object
additionalProperties: false
required: [content_kind, attachment_id, checksum]
properties:
content_kind: {const: file}
attachment_id: {type: string, format: uuid}
checksum: {type: string, pattern: "^sha256:[0-9a-f]{64}$"}
AttachmentInitRequest:
type: object
additionalProperties: false
required: [file_name, mime_type, size_bytes]
properties:
file_name: {type: string, minLength: 1, maxLength: 255}
mime_type: {type: string, minLength: 1, maxLength: 128}
size_bytes: {type: integer, minimum: 1}
ChecksumRequest:
type: object
additionalProperties: false
required: [checksum]
properties:
checksum: {type: string, pattern: "^sha256:[0-9a-f]{64}$"}
OpenLinesInbox:
type: object
additionalProperties: false
required: [event_id, event_type, external_chat_id, occurred_at]
properties:
event_id: {type: string, minLength: 1, maxLength: 255}
event_type: {type: string, enum: [message.new, dialog.closed]}
external_chat_id: {type: string, format: uuid}
bitrix_message_id: {type: [string, "null"], maxLength: 255}
occurred_at: {type: string, format: date-time}
message: {type: [object, "null"]}
VM1_app/codebase/backend/api-backend/pyproject.toml
+77
View File
@@ -0,0 +1,77 @@
[project]
name = "han-api-backend"
version = "0.1.0"
description = "HAN Chat FastAPI backend"
requires-python = ">=3.12"
dependencies = [
"alembic>=1.16,<2",
"asyncpg>=0.30,<1",
"boto3>=1.39,<2",
"fastapi>=0.116,<1",
"httpx>=0.28,<1",
"opentelemetry-api>=1.44,<2",
"opentelemetry-exporter-otlp-proto-grpc>=1.44,<2",
"opentelemetry-instrumentation-botocore>=0.65b0,<1",
"opentelemetry-instrumentation-fastapi>=0.65b0,<1",
"opentelemetry-instrumentation-httpx>=0.65b0,<1",
"opentelemetry-instrumentation-redis>=0.65b0,<1",
"opentelemetry-instrumentation-sqlalchemy>=0.65b0,<1",
"opentelemetry-sdk>=1.44,<2",
"phonenumbers>=9,<10",
"prometheus-client>=0.22,<1",
"pydantic-settings>=2.10,<3",
"pyyaml>=6,<7",
"pyjwt[crypto]>=2.10,<3",
"redis>=6,<7",
"sqlalchemy[asyncio]>=2.0.41,<3",
"structlog>=25,<26",
"uvicorn[standard]>=0.35,<1",
]
[project.optional-dependencies]
dev = [
"aiosqlite>=0.21,<1",
"mypy>=1.16,<2",
"pytest>=8.4,<9",
"pytest-asyncio>=1.0,<2",
"ruff>=0.12,<1",
"types-boto3-s3>=1.39,<2",
"types-PyYAML>=6,<7",
]
[project.scripts]
han-api = "app.main:run"
han-delivery-worker = "app.workers:delivery_main"
han-safety-worker = "app.workers:safety_main"
han-cleanup-worker = "app.workers:cleanup_main"
han-notification-expire-worker = "app.workers:notification_expire_main"
han-notification-draft-cleanup-worker = "app.workers:notification_draft_cleanup_main"
[build-system]
requires = ["hatchling"]
build-backend = "hatchling.build"
[tool.hatch.build.targets.wheel]
packages = ["app"]
[tool.pytest.ini_options]
asyncio_mode = "auto"
testpaths = ["tests"]
[tool.ruff]
target-version = "py312"
line-length = 100
[tool.ruff.lint]
select = ["E", "F", "I", "UP", "B", "ASYNC", "S"]
ignore = ["S101"]
[tool.mypy]
python_version = "3.12"
check_untyped_defs = true
warn_redundant_casts = true
warn_unused_ignores = true
ignore_missing_imports = true
disable_error_code = ["no-untyped-def", "misc", "arg-type", "assignment"]
plugins = ["pydantic.mypy", "sqlalchemy.ext.mypy.plugin"]
exclude = ["alembic/"]
VM1_app/codebase/backend/api-backend/tests/contract/test_clients.py
+249
View File
@@ -0,0 +1,249 @@
import uuid
from urllib.parse import parse_qs, urlsplit
import httpx
import pytest
from app.integrations import (
DependencyFailure,
OpenLinesClient,
S3Client,
SafetyClient,
fresh_openlines_payload,
)
from app.realtime import CHANNEL_PREFIX
from app.settings import Settings
def settings() -> Settings:
common = {
"APP_ENV": "test",
"DATABASE_URL": "postgresql+asyncpg://u:p@localhost/db",
"REDIS_URL": "redis://localhost/0",
"REDIS_REALTIME_URL": "redis://localhost/1",
"KEYCLOAK_PUBLIC_URL": "https://auth.example",
"KEYCLOAK_INTERNAL_URL": "http://keycloak:8080",
"KEYCLOAK_REALM": "han",
"KEYCLOAK_AUDIENCE": "api",
"MESSAGE_SAFETY_URL": "http://safety:8080",
"MESSAGE_SAFETY_SERVICE_TOKEN": "safety-token",
"BITRIX_LOCAL_APP_BASE_URL": "http://bitrix:8080",
"BITRIX_LOCAL_APP_INTERNAL_TOKEN": "bitrix-token",
"BITRIX_API_INBOX_TOKEN": "inbox-token",
"KEYCLOAK_SETTINGS_BRIDGE_TOKEN": "settings-token",
"SELECTEL_S3_ENDPOINT_URL": "https://s3.example",
"SELECTEL_S3_BUCKET_DOCUMENTS": "documents",
"SELECTEL_S3_BUCKET_ATTACHMENTS": "attachments",
"SELECTEL_S3_BUCKET_QUARANTINE": "quarantine",
"SELECTEL_S3_ACCESS_KEY": "access",
"SELECTEL_S3_SECRET_KEY": "secret",
"CURSOR_HMAC_SECRET": "x" * 32,
}
return Settings.model_validate(common)
def test_realtime_channel_matches_redis_acl_namespace() -> None:
assert CHANNEL_PREFIX == "han:rt:dialog:"
@pytest.mark.asyncio
async def test_s3_presigned_urls_use_virtual_hosted_addressing() -> None:
s3 = S3Client(settings())
put_url = await s3.presign_put("quarantine/users/u/file.pdf", "application/pdf", 600)
get_url = await s3.presign_get("attachments", "dialogs/d/file.pdf")
assert urlsplit(put_url).netloc == "quarantine.s3.example"
assert urlsplit(put_url).path == "/quarantine/users/u/file.pdf"
assert urlsplit(get_url).netloc == "attachments.s3.example"
assert urlsplit(get_url).path == "/dialogs/d/file.pdf"
assert parse_qs(urlsplit(put_url).query)["X-Amz-Algorithm"] == ["AWS4-HMAC-SHA256"]
assert parse_qs(urlsplit(get_url).query)["X-Amz-Algorithm"] == ["AWS4-HMAC-SHA256"]
@pytest.mark.asyncio
async def test_safety_contract_status_and_service_token() -> None:
task_id = str(uuid.uuid4())
async def handler(request: httpx.Request) -> httpx.Response:
assert request.headers["X-Service-Token"] == "safety-token"
assert request.url.path == "/internal/safety/v2/messages/check"
return httpx.Response(
202,
headers={
"Location": f"/internal/safety/v2/messages/tasks/{task_id}",
"Retry-After": "2",
},
json={
"verdict": "pending",
"processing_mode": "standard",
"config_version": 1,
"rules_version": "2026-01-01",
"task_id": task_id,
"poll_after_ms": 2000,
"expires_at": "2026-08-06T12:00:00Z",
},
)
async with httpx.AsyncClient(transport=httpx.MockTransport(handler)) as http:
result = await SafetyClient(settings(), http).check(
{"message_id": str(uuid.uuid4()), "content_kind": "text", "text": "hello"},
"request-1",
)
assert result["_status"] == 202
assert result["_location"] == f"/internal/safety/v2/messages/tasks/{task_id}"
@pytest.mark.asyncio
async def test_safety_file_body_uses_exact_attachment_schema() -> None:
attachment_id = uuid.uuid4()
async def handler(request: httpx.Request) -> httpx.Response:
body = __import__("json").loads(request.content)
assert body["attachment"] == {
"attachment_id": str(attachment_id),
"quarantine_object_key": "quarantine/users/u/file",
"quarantine_version_id": "version-1",
"quarantine_etag": '"etag-1"',
"mime_type": "application/pdf",
"size_bytes": 42,
"checksum": "sha256:" + "a" * 64,
}
assert "file" not in body
return httpx.Response(
200,
json={
"verdict": "allow",
"processing_mode": "standard",
"config_version": 1,
"rules_version": "2026-01-01",
"rule_id": "safety.all_checks_passed",
},
)
payload = {
"message_id": str(uuid.uuid4()),
"content_kind": "file",
"text": "",
"attachment": {
"attachment_id": str(attachment_id),
"quarantine_object_key": "quarantine/users/u/file",
"quarantine_version_id": "version-1",
"quarantine_etag": '"etag-1"',
"mime_type": "application/pdf",
"size_bytes": 42,
"checksum": "sha256:" + "a" * 64,
},
}
async with httpx.AsyncClient(transport=httpx.MockTransport(handler)) as http:
await SafetyClient(settings(), http).check(payload, "request-1")
@pytest.mark.asyncio
async def test_safety_auth_failure_is_dependency_failure() -> None:
async def handler(_: httpx.Request) -> httpx.Response:
return httpx.Response(401, json={"error": {"code": "service_unauthorized"}})
async with httpx.AsyncClient(transport=httpx.MockTransport(handler)) as http:
with pytest.raises(DependencyFailure):
await SafetyClient(settings(), http).check(
{"message_id": str(uuid.uuid4()), "content_kind": "text", "text": "hello"},
"request-1",
)
@pytest.mark.asyncio
async def test_safety_poll_uses_location_and_rejects_untrusted_location() -> None:
task_id = str(uuid.uuid4())
async def handler(request: httpx.Request) -> httpx.Response:
assert request.url.path == f"/internal/safety/v2/messages/tasks/{task_id}"
return httpx.Response(
403,
json={
"verdict": "deny",
"processing_mode": "standard",
"config_version": 2,
"rules_version": "2026-08-06",
"rule_id": "file.malware_detected",
"reason_code": "message_blocked",
},
)
async with httpx.AsyncClient(transport=httpx.MockTransport(handler)) as http:
client = SafetyClient(settings(), http)
result = await client.poll(
f"/internal/safety/v2/messages/tasks/{task_id}", "request-1"
)
assert result["_status"] == 403
with pytest.raises(DependencyFailure, match="invalid_safety_location"):
await client.poll("https://attacker.example/task-1", "request-1")
@pytest.mark.asyncio
async def test_safety_fails_closed_on_malformed_success() -> None:
async def handler(_: httpx.Request) -> httpx.Response:
return httpx.Response(200, json={"verdict": "allow"})
async with httpx.AsyncClient(transport=httpx.MockTransport(handler)) as http:
with pytest.raises(DependencyFailure, match="invalid_safety_response"):
await SafetyClient(settings(), http).check(
{"message_id": str(uuid.uuid4()), "content_kind": "text", "text": "hello"},
"request-1",
)
@pytest.mark.asyncio
async def test_openlines_contract_uses_bearer_and_idempotency() -> None:
message_id = uuid.uuid4()
async def handler(request: httpx.Request) -> httpx.Response:
assert request.headers["Authorization"] == "Bearer bitrix-token"
assert request.headers["Idempotency-Key"] == str(message_id)
assert request.url.path == "/internal/openlines/v1/messages"
return httpx.Response(201, json={"status": "accepted"})
async with httpx.AsyncClient(transport=httpx.MockTransport(handler)) as http:
result = await OpenLinesClient(settings(), http).send(
message_id, {"message_id": str(message_id)}, "request-1"
)
assert result["status"] == "accepted"
@pytest.mark.asyncio
async def test_openlines_payload_gets_fresh_download_url_without_storage_fields() -> None:
class FakeS3:
calls = 0
async def presign_get(self, bucket: str, key: str, ttl: int = 300) -> str:
self.calls += 1
return f"https://download.example/{bucket}/{key}?generation={self.calls}"
payload = {
"message_id": str(uuid.uuid4()),
"external_chat_id": str(uuid.uuid4()),
"occurred_at": "2026-07-10T12:00:00+00:00",
"user": {"id": str(uuid.uuid4()), "display_name": "+79990000000"},
"message": {
"content_kind": "file",
"text": "",
"files": [{
"attachment_id": str(uuid.uuid4()),
"name": "file.pdf",
"mime_type": "application/pdf",
"size_bytes": 42,
"_storage_bucket": "attachments",
"_object_key": "dialogs/d/file",
}],
},
}
s3 = FakeS3()
first = await fresh_openlines_payload(payload, s3) # type: ignore[arg-type]
second = await fresh_openlines_payload(payload, s3) # type: ignore[arg-type]
assert first["occurred_at"] and first["user"]["id"]
assert first["message"]["content_kind"] == "file"
assert (
first["message"]["files"][0]["download_url"]
!= second["message"]["files"][0]["download_url"]
)
assert all(not key.startswith("_") for key in first["message"]["files"][0])
VM1_app/codebase/backend/api-backend/tests/contract/test_openapi.py
+226
View File
@@ -0,0 +1,226 @@
import asyncio
import base64
import json
from pathlib import Path
from types import SimpleNamespace
import pytest
import yaml
from alembic.config import Config
from alembic.script import ScriptDirectory
from pydantic import SecretStr
from app.main import (
EXPECTED_API_DB_REVISION,
app,
otp_settings,
refresh_jwks_cache,
websocket_token,
)
from app.services import SettingsSnapshot
EXPECTED_PATHS = {
"/health/live",
"/health/ready",
"/api/v1/public/app-config",
"/api/v1/public/content",
"/api/v1/public/notifications",
"/api/v1/public/notification-types",
"/api/v1/auth/bootstrap",
"/api/v1/consents",
"/api/v1/analytics/session-start",
"/api/v1/me",
"/api/v1/me/documents",
"/api/v1/notifications",
"/api/v1/notifications/counter",
"/api/v1/notifications/{notification_id}",
"/api/v1/notifications/{notification_id}/read",
"/api/v1/notifications/{notification_id}/hide",
"/api/v1/notifications/{notification_id}/buttons/{button_code}",
"/api/v1/notifications/{notification_id}/cta",
"/api/v1/notifications/{notification_id}/documents/{document_id}/download-url",
"/api/v1/uploads/init",
"/api/v1/uploads/{draft_id}/complete",
"/api/v1/uploads",
"/api/v1/uploads/{draft_id}",
"/api/v1/documents/{document_id}",
"/api/v1/documents/{document_id}/download-url",
"/api/v1/dialogs",
"/api/v1/dialogs/{dialog_id}",
"/api/v1/dialogs/{dialog_id}/messages",
"/api/v1/dialogs/{dialog_id}/attachments/init",
"/api/v1/dialogs/{dialog_id}/attachments/{attachment_id}/complete",
"/api/v1/dialogs/{dialog_id}/attachments/{attachment_id}/download-url",
"/internal/openlines/v1/inbox",
"/internal/notifications/v1/notifications",
"/internal/notifications/v1/notifications/cancel",
"/internal/settings/v1/otp",
}
def test_openapi_31_contains_all_http_contracts() -> None:
schema = app.openapi()
assert schema["openapi"].startswith("3.1.")
assert EXPECTED_PATHS <= schema["paths"].keys()
assert all(not path.startswith("/internal/safety") for path in schema["paths"])
committed = yaml.safe_load(Path("openapi.yaml").read_text(encoding="utf-8"))
assert committed["openapi"] == "3.1.0"
assert committed["paths"].keys() == schema["paths"].keys()
def test_readiness_expected_revision_matches_alembic_head() -> None:
scripts = ScriptDirectory.from_config(Config("alembic.ini"))
assert EXPECTED_API_DB_REVISION == scripts.get_current_head()
@pytest.mark.asyncio
async def test_jwks_refresh_loop_recovers_after_startup_race(monkeypatch) -> None:
class FakeJWKS:
has_keys = False
refresh_calls = 0
async def refresh(self) -> None:
self.refresh_calls += 1
self.has_keys = True
jwks = FakeJWKS()
test_app = SimpleNamespace(
state=SimpleNamespace(
jwks=jwks,
settings=SimpleNamespace(jwks_cache_ttl_seconds=300),
)
)
delays: list[int] = []
async def fake_sleep(delay: int) -> None:
delays.append(delay)
if len(delays) > 1:
raise asyncio.CancelledError
monkeypatch.setattr("app.main.asyncio.sleep", fake_sleep)
with pytest.raises(asyncio.CancelledError):
await refresh_jwks_cache(test_app)
assert jwks.refresh_calls == 1
assert delays == [5, 300]
def test_websocket_route_is_registered() -> None:
assert any(getattr(route, "path", None) == "/api/v1/realtime" for route in app.routes)
def test_notification_http_methods_match_contract() -> None:
paths = app.openapi()["paths"]
expected = {
"/api/v1/public/notifications": {"get"},
"/api/v1/public/notification-types": {"get"},
"/api/v1/notifications": {"get"},
"/api/v1/notifications/counter": {"get"},
"/api/v1/notifications/{notification_id}": {"get"},
"/api/v1/notifications/{notification_id}/read": {"post"},
"/api/v1/notifications/{notification_id}/hide": {"post"},
"/api/v1/notifications/{notification_id}/buttons/{button_code}": {"post"},
"/api/v1/notifications/{notification_id}/cta": {"post"},
(
"/api/v1/notifications/{notification_id}/documents/"
"{document_id}/download-url"
): {"get"},
"/api/v1/uploads/init": {"post"},
"/api/v1/uploads/{draft_id}/complete": {"post"},
"/api/v1/uploads": {"get"},
"/api/v1/uploads/{draft_id}": {"delete"},
"/internal/notifications/v1/notifications": {"post"},
"/internal/notifications/v1/notifications/cancel": {"post"},
}
for path, methods in expected.items():
assert methods <= paths[path].keys()
def test_websocket_accepts_canonical_base64url_jwt_protocol() -> None:
jwt = "header.payload.signature"
encoded = base64.urlsafe_b64encode(jwt.encode()).decode().rstrip("=")
websocket = SimpleNamespace(
headers={"sec-websocket-protocol": f"han-chat-v1, han.jwt.{encoded}"},
query_params={},
)
assert websocket_token(websocket) == (jwt, f"han.jwt.{encoded}")
def test_committed_openapi_server_does_not_double_api_prefix() -> None:
committed = yaml.safe_load(Path("openapi.yaml").read_text(encoding="utf-8"))
assert committed["servers"] == [{"url": "/"}]
def test_public_config_contract_exposes_message_length() -> None:
committed = yaml.safe_load(Path("openapi.yaml").read_text(encoding="utf-8"))
response = committed["paths"]["/api/v1/public/app-config"]["get"]["responses"]["200"]
messages = response["content"]["application/json"]["schema"]["properties"]["messages"]
assert messages["required"] == ["max_text_length"]
assert messages["properties"]["max_text_length"]["maximum"] == 10_000
def test_otp_settings_contract_is_strict_and_complete() -> None:
generated = app.openapi()
response = generated["paths"]["/internal/settings/v1/otp"]["get"]["responses"]["200"]
schema_ref = response["content"]["application/json"]["schema"]["$ref"]
schema = generated["components"]["schemas"][schema_ref.rsplit("/", 1)[-1]]
assert set(schema["required"]) == {
"max_send_attempts_per_24h",
"min_seconds_between_attempts",
"max_verify_attempts",
"code_length",
"ttl_seconds",
"sms_order_timeout_ms",
"version",
"cache_ttl_seconds",
}
assert schema["additionalProperties"] is False
assert schema["properties"]["code_length"] == {
"type": "integer",
"maximum": 10.0,
"minimum": 4.0,
"title": "Code Length",
}
assert schema["properties"]["ttl_seconds"]["multipleOf"] == 60
async def test_otp_settings_returns_runtime_values_and_supports_etag() -> None:
request = SimpleNamespace(
headers={"Authorization": "Bearer bridge-token"},
app=SimpleNamespace(
state=SimpleNamespace(
settings=SimpleNamespace(
keycloak_settings_bridge_token=SecretStr("bridge-token")
)
)
),
)
settings = SettingsSnapshot(
{
"otp.phone.max_send_attempts_per_24h": "3",
"otp.phone.min_seconds_between_attempts": "30",
"otp.phone.max_verify_attempts": "5",
"otp.phone.code_length": "6",
"otp.phone.ttl_seconds": "60",
"otp.phone.sms_order_timeout_ms": "3000",
},
"settings-version",
)
response = await otp_settings(request, settings)
assert json.loads(response.body) == {
"max_send_attempts_per_24h": 3,
"min_seconds_between_attempts": 30,
"max_verify_attempts": 5,
"code_length": 6,
"ttl_seconds": 60,
"sms_order_timeout_ms": 3000,
"version": "settings-version",
"cache_ttl_seconds": 60,
}
cached = await otp_settings(request, settings, response.headers["etag"])
assert cached.status_code == 304
VM1_app/codebase/backend/api-backend/tests/unit/test_audit_context.py
+153
View File
@@ -0,0 +1,153 @@
import hashlib
import uuid
from types import SimpleNamespace
import pytest
from starlette.requests import Request
from app.main import (
app,
client_ip,
request_trace_id,
required_user_audit_context,
user_agent_hash,
)
from app.schemas import Device
from app.services import AuditContext, DomainError, audit, device_snapshot
def request(
*,
peer: str = "172.18.0.5",
forwarded: str | None = None,
traceparent: str | None = None,
user_agent: str | None = None,
ux_session_id: str | None = None,
trusted: str = "172.16.0.0/12",
) -> Request:
headers: list[tuple[bytes, bytes]] = []
for name, value in (
("x-forwarded-for", forwarded),
("traceparent", traceparent),
("user-agent", user_agent),
("x-ux-session-id", ux_session_id),
):
if value is not None:
headers.append((name.encode(), value.encode()))
settings = SimpleNamespace(trusted_proxy_cidrs=trusted)
app = SimpleNamespace(state=SimpleNamespace(settings=settings))
return Request(
{
"type": "http",
"method": "GET",
"path": "/",
"headers": headers,
"client": (peer, 12345),
"server": ("test", 443),
"scheme": "https",
"query_string": b"",
"app": app,
}
)
def test_client_ip_only_trusts_forwarded_header_from_configured_proxy() -> None:
assert client_ip(request(forwarded="203.0.113.10")) == "203.0.113.10"
assert (
client_ip(request(peer="198.51.100.7", forwarded="203.0.113.10"))
== "198.51.100.7"
)
assert client_ip(request(peer="not-an-ip")) is None
def test_trace_id_uses_valid_w3c_header_and_generates_fallback() -> None:
trace_id = "1" * 32
assert request_trace_id(request(traceparent=f"00-{trace_id}-{'2' * 16}-01")) == trace_id
fallback = request_trace_id(request(traceparent="invalid"))
assert len(fallback) == 32
int(fallback, 16)
def test_user_agent_is_hashed_and_device_parameters_are_preserved() -> None:
agent = "Example Browser/1.0"
assert user_agent_hash(request(user_agent=agent)) == hashlib.sha256(agent.encode()).hexdigest()
snapshot = device_snapshot(
Device(platform="web", app_version="1.2.3", device_id="raw-device-id")
)
assert snapshot == {
"platform": "web",
"app_version": "1.2.3",
"device_id": "raw-device-id",
}
def test_audit_copies_request_context_and_bounded_metadata() -> None:
session_id = uuid.uuid4()
user_id = uuid.uuid4()
context = AuditContext(
request_id=str(uuid.uuid4()),
trace_id="a" * 32,
ux_session_id=session_id,
user_agent_hash="b" * 64,
client_ip="203.0.113.10",
)
event = audit(
"dialog.created",
context,
user_id,
"dialog",
uuid.uuid4(),
metadata={"status": "open"},
)
assert event.user_id == user_id
assert event.ux_session_id == session_id
assert event.trace_id == "a" * 32
assert event.user_agent_hash == "b" * 64
assert event.metadata_json == {"status": "open"}
assert event.outcome == "success"
assert not hasattr(event, "ip")
failed = audit("message.failed", context, user_id, outcome="failed")
assert failed.outcome == "failed"
def test_post_session_routes_publish_required_ux_header_in_openapi() -> None:
operation = app.openapi()["paths"]["/api/v1/dialogs"]["post"]
ux_header = next(
parameter
for parameter in operation["parameters"]
if parameter["name"] == "X-Ux-Session-Id"
)
assert ux_header["in"] == "header"
assert ux_header["required"] is True
@pytest.mark.asyncio
async def test_user_audit_context_requires_existing_owned_session() -> None:
session_id = uuid.uuid4()
user = SimpleNamespace(id=uuid.uuid4())
incoming = request(
forwarded="203.0.113.10",
ux_session_id=str(session_id),
user_agent="Example Browser/1.0",
)
incoming.state.request_id = str(uuid.uuid4())
incoming.state.trace_id = "c" * 32
incoming.state.user_agent_hash = user_agent_hash(incoming)
class Db:
async def scalar(self, _query):
return session_id
context = await required_user_audit_context(incoming, Db(), user, str(session_id))
assert context.ux_session_id == session_id
assert context.client_ip == "203.0.113.10"
missing = request(forwarded="203.0.113.10")
missing.state.request_id = str(uuid.uuid4())
missing.state.trace_id = "d" * 32
missing.state.user_agent_hash = None
with pytest.raises(DomainError, match="X-Ux-Session-Id is required"):
await required_user_audit_context(missing, Db(), user, "")
VM1_app/codebase/backend/api-backend/tests/unit/test_cli_settings.py
+78
View File
@@ -0,0 +1,78 @@
from pathlib import Path
import pytest
from app.cli.seed_settings import load_seed
from app.services import REQUIRED_SETTINGS
def test_production_like_seed_contains_all_mandatory_settings() -> None:
path = Path(__file__).resolve().parents[3] / "deployment/app-settings.production-like.yaml"
rows = load_seed(path)
assert REQUIRED_SETTINGS <= {row["setting_key"] for row in rows}
assert all(row["record_status"] == "A" for row in rows)
values = {row["setting_key"]: row["setting_value"] for row in rows}
assert values["otp.phone.code_length"] == "6"
assert values["otp.phone.ttl_seconds"] == "60"
assert values["otp.phone.sms_order_timeout_ms"] == "3000"
assert values["chat.message.max_length"] == "4000"
def test_seed_rejects_invalid_typed_value(tmp_path: Path) -> None:
path = tmp_path / "settings.yaml"
path.write_text(
"schema_version: 1\nsettings:\n"
" bad.integer: {type: integer, value: nope, public: false}\n",
encoding="utf-8",
)
with pytest.raises(ValueError, match="integer value expected"):
load_seed(path)
@pytest.mark.parametrize(
("key", "value", "message"),
[
("otp.phone.code_length", 3, "between 4 and 10"),
("otp.phone.ttl_seconds", 61, "divisible by 60"),
("otp.phone.sms_order_timeout_ms", 0, "must be positive"),
],
)
def test_seed_rejects_invalid_otp_settings(
tmp_path: Path, key: str, value: int, message: str
) -> None:
path = tmp_path / "settings.yaml"
path.write_text(
"schema_version: 1\nsettings:\n"
f" {key}: {{type: integer, value: {value}, public: false}}\n",
encoding="utf-8",
)
with pytest.raises(ValueError, match=message):
load_seed(path)
def test_seed_rejects_public_otp_setting(tmp_path: Path) -> None:
path = tmp_path / "settings.yaml"
path.write_text(
"schema_version: 1\nsettings:\n"
" otp.phone.code_length: {type: integer, value: 6, public: true}\n",
encoding="utf-8",
)
with pytest.raises(ValueError, match="must not be public"):
load_seed(path)
@pytest.mark.parametrize("value", [0, 10001])
def test_seed_rejects_invalid_chat_message_max_length(tmp_path: Path, value: int) -> None:
path = tmp_path / "settings.yaml"
path.write_text(
"schema_version: 1\nsettings:\n"
f" chat.message.max_length: {{type: integer, value: {value}, public: true}}\n",
encoding="utf-8",
)
with pytest.raises(ValueError, match="value must be between 1 and 10000"):
load_seed(path)
VM1_app/codebase/backend/api-backend/tests/unit/test_domain.py
+104
View File
@@ -0,0 +1,104 @@
import uuid
import pytest
from pydantic import TypeAdapter, ValidationError
from app.auth import canonical_phone
from app.integrations import CircuitBreaker, RateLimiter
from app.postgres import asyncpg_dsn
from app.schemas import (
FileMessageRequest,
MessageRequest,
TextMessageRequest,
canonical_fingerprint,
decode_cursor,
encode_cursor,
)
from app.services import MESSAGE_SAFETY_REPLIES, safety_reply_message
def test_asyncpg_receives_libpq_dsn_without_sqlalchemy_driver() -> None:
url = (
"postgresql+asyncpg://user:password@db:5433/han_chat"
"?sslmode=verify-full&sslrootcert=/run/secrets/pg-ca.pem"
)
assert asyncpg_dsn(url) == url.replace("postgresql+asyncpg://", "postgresql://", 1)
def test_phone_claim_priority_and_e164_validation() -> None:
claims = {"phone_number": "+74999591007", "preferred_username": "+12025550123"}
assert canonical_phone(claims) == "+74999591007"
assert canonical_phone({"phone_number": "89999999999"}) is None
def test_message_discriminated_union() -> None:
adapter = TypeAdapter(MessageRequest)
assert isinstance(
adapter.validate_python({"content_kind": "text", "text": "Здравствуйте"}),
TextMessageRequest,
)
assert isinstance(
adapter.validate_python(
{
"content_kind": "file",
"attachment_id": str(uuid.uuid4()),
"checksum": "sha256:" + "a" * 64,
}
),
FileMessageRequest,
)
with pytest.raises(ValidationError):
adapter.validate_python(
{"content_kind": "text", "text": "", "attachment_id": str(uuid.uuid4())}
)
longest = adapter.validate_python({"content_kind": "text", "text": "а" * 10_000})
assert len(longest.text) == 10_000
with pytest.raises(ValidationError):
adapter.validate_python({"content_kind": "text", "text": "а" * 10_001})
def test_message_safety_business_replies_are_content_specific() -> None:
assert "переформулировать" in MESSAGE_SAFETY_REPLIES["text"]
assert "документ" in MESSAGE_SAFETY_REPLIES["file"]
reply = safety_reply_message(uuid.uuid4(), "text")
assert reply.sender_type == "company"
assert reply.safety_status == "allowed"
assert reply.delivery_status == "delivered"
def test_fingerprint_is_canonical_and_user_scoped() -> None:
user = uuid.uuid4()
first = canonical_fingerprint("post", "/dialogs/{id}", {"id": "1"}, {"b": 2, "a": 1}, user)
second = canonical_fingerprint("POST", "/dialogs/{id}", {"id": "1"}, {"a": 1, "b": 2}, user)
assert first == second
assert first != canonical_fingerprint(
"POST", "/dialogs/{id}", {"id": "1"}, {"a": 1, "b": 2}, uuid.uuid4()
)
def test_cursor_roundtrip_and_tamper_rejection() -> None:
secret = b"test-secret" * 4
cursor = encode_cursor({"id": str(uuid.uuid4()), "created_at": "2026-01-01"}, secret)
assert decode_cursor(cursor, secret)["created_at"] == "2026-01-01"
with pytest.raises(ValueError, match="invalid cursor"):
decode_cursor(cursor[:-2] + "aa", secret)
def test_rate_limit_keys_do_not_expose_identity() -> None:
key = RateLimiter.key("ip", "203.0.113.7", "public", 60)
assert "203.0.113.7" not in key
assert key.startswith("han:api:rl:ip:")
def test_circuit_breaker_opens_and_half_opens(monkeypatch: pytest.MonkeyPatch) -> None:
clock = [10.0]
monkeypatch.setattr("app.integrations.time.monotonic", lambda: clock[0])
breaker = CircuitBreaker(2, 30)
breaker.failure()
breaker.failure()
assert not breaker.allow()
clock[0] = 41.0
assert breaker.allow()
breaker.success()
assert breaker.allow()
VM1_app/codebase/backend/api-backend/tests/unit/test_logging_security.py
+31
View File
@@ -0,0 +1,31 @@
from app.logging_security import REDACTED, redact_event, sanitize_text
def test_redacts_sensitive_fields_recursively() -> None:
event = {
"authorization": "Bearer top-secret",
"nested": {
"database_url": "postgresql://user:password@db/app",
"safe": "kept",
},
}
redacted = redact_event(None, "info", event)
assert redacted["authorization"] == REDACTED
assert redacted["nested"]["database_url"] == REDACTED
assert redacted["nested"]["safe"] == "kept"
def test_redacts_credentials_embedded_in_text() -> None:
value = (
"POST https://callback-user:callback-password@example.test/cb"
"?token=query-secret Authorization=Bearer header-secret"
)
redacted = sanitize_text(value)
assert "callback-password" not in redacted
assert "query-secret" not in redacted
assert "header-secret" not in redacted
assert redacted.count(REDACTED) == 3
VM1_app/codebase/backend/api-backend/tests/unit/test_notifications.py
+589
View File
@@ -0,0 +1,589 @@
import ast
import json
import uuid
from collections.abc import AsyncGenerator
from datetime import UTC, datetime, timedelta
from pathlib import Path
from types import SimpleNamespace
from typing import Any, cast
from unittest.mock import AsyncMock, Mock
import pytest
from pydantic import ValidationError
from app.db import Document
from app.notification_models import (
ClientUploadDraft,
Notification,
NotificationButton,
NotificationDocument,
NotificationSource,
NotificationType,
uuid7,
)
from app.notification_schemas import NotificationCreateRequest
from app.notification_service import (
_apply_hidden_ttl,
apply_read_or_hide,
authenticate_source,
catalog,
create_notification,
document_download,
expire_notifications,
fingerprint,
invoke_cta_state,
press_button,
source_token_hash,
validate_create,
)
from app.realtime import USER_CHANNEL_PREFIX, RealtimeFanout
from app.services import AuditContext, DomainError, SettingsSnapshot
from app.workers import notification_draft_cleanup_once
def test_notification_migration_does_not_prepare_multiple_sql_commands() -> None:
migration = Path("alembic/versions/0008_notification_center_v1.py")
tree = ast.parse(migration.read_text(encoding="utf-8"))
for call in (node for node in ast.walk(tree) if isinstance(node, ast.Call)):
if (
isinstance(call.func, ast.Attribute)
and call.func.attr == "execute"
and call.args
and isinstance(call.args[0], ast.Call)
and isinstance(call.args[0].func, ast.Attribute)
and call.args[0].func.attr == "text"
and call.args[0].args
and isinstance(call.args[0].args[0], ast.Constant)
and isinstance(call.args[0].args[0].value, str)
):
assert call.args[0].args[0].value.count(";") <= 1
def create_body(**changes: object) -> NotificationCreateRequest:
values: dict[str, object] = {
"user_id": uuid.uuid4(),
"notification_type": "news",
"source": "producer_test",
"external_id": "event-1",
"notification_datetime": "2026-07-27T12:00:00Z",
"header": "Новость",
"details": {"details_text": "Текст"},
}
values.update(changes)
return NotificationCreateRequest.model_validate(values)
def context() -> AuditContext:
return AuditContext("request-1", "trace-1", None, None, None)
def snapshot(default_ttl: int = 3) -> SettingsSnapshot:
return SettingsSnapshot(
{
"notification.hidden.default_ttl_days": str(default_ttl),
"notification.center.max_items": "15",
},
"v1",
)
def notification(**changes: object) -> Notification:
values: dict[str, object] = {
"id": uuid.uuid4(),
"user_id": uuid.uuid4(),
"notification_type": "news",
"source": "producer_test",
"external_id": "event-1",
"request_fingerprint": "a" * 64,
"notification_datetime": datetime.now(UTC),
"header": "Header",
"lifecycle_status": "active",
"visibility": "visible",
"is_read": False,
"date_expired": None,
"close_reason": None,
"closed_at": None,
}
values.update(changes)
return Notification(**values)
def kind(**changes: object) -> NotificationType:
values: dict[str, object] = {
"id": uuid.uuid4(),
"code": "news",
"contour": "P",
"priority": 4,
"countable": True,
"label": "Новость",
"color_token": "info",
"icon_code": "news",
"cta_text": "Подробнее",
"cta_action": "open_detail",
"cta_sets_hidden": False,
"cta_close_reason": None,
"button_primary_code": "gotit",
"button_secondary_code": None,
"hidden_ttl_days": None,
"documents_allowed": False,
"hide_on_document_download": False,
"required_detail_blocks": [],
}
values.update(changes)
return NotificationType(**values)
class ScalarRows:
def __init__(self, rows: list[object]) -> None:
self.rows = rows
def scalars(self) -> "ScalarRows":
return self
def all(self) -> list[object]:
return self.rows
def test_uuid7_has_rfc_version_variant_and_embedded_timestamp(
monkeypatch: pytest.MonkeyPatch,
) -> None:
timestamp_ns = 1_722_340_800_123_000_000
monkeypatch.setattr("app.notification_models.time.time_ns", lambda: timestamp_ns)
monkeypatch.setattr("app.notification_models.secrets.randbits", lambda bits: (1 << bits) - 1)
value = uuid7()
assert value.version == 7
assert value.variant == uuid.RFC_4122
assert value.int >> 80 == timestamp_ns // 1_000_000
def test_fingerprint_is_canonical_stable_and_sensitive_to_body() -> None:
first = create_body()
same = NotificationCreateRequest.model_validate(first.model_dump(mode="json"))
changed = create_body(header="Другая новость")
assert fingerprint(first) == fingerprint(same)
assert fingerprint(first) != fingerprint(changed)
assert len(fingerprint(first)) == 64
def test_create_schema_rejects_read_only_or_unknown_detail_blocks() -> None:
with pytest.raises(ValidationError) as pending:
create_body(details={"details_text": "Text", "pending_documents": []})
with pytest.raises(ValidationError) as unknown:
create_body(details={"details_text": "Text", "invented": True})
assert "pending_documents" in str(pending.value)
assert "invented" in str(unknown.value)
def test_catalog_driven_create_validation_covers_required_and_forbidden_fields() -> None:
action = SimpleNamespace(required_instance_fields=["details"])
docs_kind = kind(
required_detail_blocks=["documents"],
documents_allowed=True,
button_primary_code="gotit",
)
with pytest.raises(DomainError) as error:
validate_create(create_body(details={"details_text": "No documents"}), docs_kind, action)
assert error.value.code == "validation_error"
assert error.value.details["fields"] == ["details.documents"]
with pytest.raises(DomainError) as forbidden:
validate_create(
create_body(
details={"details_text": "Text"},
payment_url="https://pay.example/order",
),
kind(),
action,
)
assert forbidden.value.details["fields"] == ["payment_url"]
@pytest.mark.asyncio
async def test_duplicate_create_returns_existing_only_for_matching_fingerprint(
monkeypatch: pytest.MonkeyPatch,
) -> None:
body = create_body()
existing = notification(request_fingerprint=fingerprint(body))
session = SimpleNamespace(
scalar=AsyncMock(return_value=existing),
add=Mock(),
commit=AsyncMock(),
)
dto = {"id": str(existing.id)}
monkeypatch.setattr(
"app.notification_service.notification_dto", AsyncMock(return_value=dto)
)
result, status = await create_notification(
session,
body,
SimpleNamespace(code="producer_test"),
SimpleNamespace(),
SimpleNamespace(),
context(),
)
assert (result, status) == (dto, 200)
existing.request_fingerprint = "0" * 64
with pytest.raises(DomainError) as conflict:
await create_notification(
session,
body,
SimpleNamespace(code="producer_test"),
SimpleNamespace(),
SimpleNamespace(),
context(),
)
assert conflict.value.code == "notification_conflict"
assert conflict.value.status == 409
assert conflict.value.details == {"notification_id": str(existing.id)}
def test_hidden_ttl_preserves_existing_expiry_and_uses_type_or_default(
monkeypatch: pytest.MonkeyPatch,
) -> None:
now = datetime(2026, 7, 27, 12, tzinfo=UTC)
monkeypatch.setattr("app.notification_service.datetime", SimpleNamespace(now=lambda tz: now))
existing = now + timedelta(hours=2)
with_expiry = notification(date_expired=existing)
type_specific = notification()
defaulted = notification()
_apply_hidden_ttl(with_expiry, kind(hidden_ttl_days=9), snapshot())
_apply_hidden_ttl(type_specific, kind(hidden_ttl_days=5), snapshot())
_apply_hidden_ttl(defaulted, kind(hidden_ttl_days=None), snapshot(3))
assert with_expiry.date_expired == existing
assert type_specific.date_expired == now + timedelta(days=5)
assert defaulted.date_expired == now + timedelta(days=3)
assert {with_expiry.visibility, type_specific.visibility, defaulted.visibility} == {"hidden"}
@pytest.mark.asyncio
async def test_hide_applies_ttl_without_marking_read(
monkeypatch: pytest.MonkeyPatch,
) -> None:
item = notification()
monkeypatch.setattr(
"app.notification_service.owned_notification",
AsyncMock(return_value=(item, kind(hidden_ttl_days=2))),
)
monkeypatch.setattr("app.notification_service.unread_count", AsyncMock(return_value=1))
session = SimpleNamespace(add=Mock(), commit=AsyncMock())
fanout = SimpleNamespace(publish_user=AsyncMock())
result = await apply_read_or_hide(
session,
item.user_id,
item.id,
"hide",
snapshot(),
fanout,
context(),
)
assert result["visibility"] == "hidden"
assert result["is_read"] is False
assert item.date_expired is not None
event = fanout.publish_user.await_args.args[1]
assert event["date_expired"] == item.date_expired
@pytest.mark.asyncio
async def test_cta_and_buttons_follow_catalog_lifecycle(
monkeypatch: pytest.MonkeyPatch,
) -> None:
item = notification()
cta_kind = kind(cta_action="open_detail")
monkeypatch.setattr(
"app.notification_service.owned_notification",
AsyncMock(return_value=(item, cta_kind)),
)
monkeypatch.setattr("app.notification_service.unread_count", AsyncMock(return_value=0))
session = SimpleNamespace(add=Mock(), commit=AsyncMock(), scalar=AsyncMock())
fanout = SimpleNamespace(publish_user=AsyncMock())
_, _, cta_result = await invoke_cta_state(
session, item.user_id, item.id, snapshot(), fanout, context()
)
assert item.is_read is True
assert item.visibility == "visible"
assert item.lifecycle_status == "active"
assert cta_result["result"]["action"] == "open_detail"
button = NotificationButton(
id=uuid.uuid4(),
code="done",
label="Готово",
sets_hidden=False,
applies_hidden_ttl=False,
close_reason="user_done",
submits_documents=False,
)
session.scalar.return_value = button
cta_kind.button_primary_code = "done"
result = await press_button(
session, item.user_id, item.id, "done", snapshot(), fanout, context()
)
assert result["lifecycle_status"] == "closed"
assert result["close_reason"] == "user_done"
assert item.closed_at is not None
@pytest.mark.asyncio
async def test_first_download_of_any_document_hides_once_and_preserves_expiry(
monkeypatch: pytest.MonkeyPatch,
) -> None:
expires = datetime.now(UTC) + timedelta(hours=1)
item = notification(date_expired=expires)
download_kind = kind(
code="docs_ready",
documents_allowed=True,
hide_on_document_download=True,
)
link = NotificationDocument(
id=uuid.uuid4(),
notification_id=item.id,
document_id=uuid.uuid4(),
sort_order=0,
download_url_issued_at=None,
)
document = Document(
id=link.document_id,
user_id=item.user_id,
name="result.pdf",
mime_type="application/pdf",
size_bytes=42,
checksum_sha256="a" * 64,
storage_bucket="documents",
object_key="documents/result.pdf",
sent_at=datetime.now(UTC),
)
row_result = SimpleNamespace(one_or_none=lambda: (link, document))
session = SimpleNamespace(
execute=AsyncMock(return_value=row_result),
scalar=AsyncMock(return_value=0),
add=Mock(),
commit=AsyncMock(),
)
monkeypatch.setattr(
"app.notification_service.owned_notification",
AsyncMock(return_value=(item, download_kind)),
)
monkeypatch.setattr("app.notification_service.unread_count", AsyncMock(return_value=0))
fanout = SimpleNamespace(publish_user=AsyncMock())
s3 = SimpleNamespace(presign_get=AsyncMock(return_value="https://download.example/file"))
result = await document_download(
session,
item.user_id,
item.id,
document.id,
snapshot(),
s3,
fanout,
context(),
)
assert result["download_url"] == "https://download.example/file"
assert item.is_read is True
assert item.visibility == "hidden"
assert item.date_expired == expires
assert link.download_url_issued_at is not None
fanout.publish_user.assert_awaited_once()
@pytest.mark.asyncio
async def test_source_token_hash_and_producer_authentication() -> None:
value = "producer-secret"
source = NotificationSource(
id=uuid.uuid4(),
code="producer_test",
description="test",
token_hash=source_token_hash(value),
token_rotated_at=None,
record_status="A",
)
session = SimpleNamespace(execute=AsyncMock(return_value=ScalarRows([source])))
assert await authenticate_source(session, f"Bearer {value}") is source
assert source.token_hash != value
with pytest.raises(DomainError) as invalid:
await authenticate_source(session, "Bearer wrong")
assert (invalid.value.code, invalid.value.status) == ("unauthorized", 401)
@pytest.mark.asyncio
async def test_catalog_dto_is_sorted_and_contains_only_public_behavior() -> None:
types = [
kind(code="news", contour="P", priority=4, button_primary_code="gotit"),
kind(
code="urgent",
contour="P",
priority=1,
label="Срочно",
button_primary_code="done",
button_secondary_code="later",
),
]
buttons = [
NotificationButton(id=uuid.uuid4(), code="done", label="Готово"),
NotificationButton(id=uuid.uuid4(), code="later", label="Позже"),
NotificationButton(id=uuid.uuid4(), code="gotit", label="Понятно"),
]
session = SimpleNamespace(
execute=AsyncMock(side_effect=[ScalarRows(types), ScalarRows(buttons)])
)
result = await catalog(session)
assert [item["code"] for item in result] == ["urgent", "news"]
assert result[0]["button_primary"] == {"code": "done", "label": "Готово"}
assert result[0]["button_secondary"] == {"code": "later", "label": "Позже"}
assert not {
"hidden_ttl_days",
"cta_sets_hidden",
"cta_close_reason",
"required_detail_blocks",
} & result[0].keys()
button_statement = session.execute.await_args_list[1].args[0]
assert "notification_buttons.record_status" in str(button_statement)
@pytest.mark.asyncio
async def test_realtime_uses_per_user_channel_and_strips_internal_identity() -> None:
user_id = uuid.uuid4()
class PubSub:
def __init__(self) -> None:
self.channels: tuple[str, ...] = ()
async def subscribe(self, *channels: str) -> None:
self.channels = channels
async def get_message(self, **_kwargs: object) -> dict[str, str]:
return {
"data": json.dumps(
{
"type": "notification.updated",
"_user_id": str(user_id),
"notification_id": str(uuid.uuid4()),
}
)
}
async def unsubscribe(self, *_channels: str) -> None:
return None
async def aclose(self) -> None:
return None
pubsub = PubSub()
redis = SimpleNamespace(publish=AsyncMock(), pubsub=lambda: pubsub)
fanout = RealtimeFanout(redis)
await fanout.publish_user(user_id, {"type": "notification.created"})
channel, payload = redis.publish.await_args.args
assert channel == USER_CHANNEL_PREFIX + str(user_id)
assert json.loads(payload)["_user_id"] == str(user_id)
stream = cast(
AsyncGenerator[dict[str, Any], None],
fanout.events(set(), user_id, notifications=True),
)
event = await anext(stream)
await stream.aclose()
assert pubsub.channels == (USER_CHANNEL_PREFIX + str(user_id),)
assert event["type"] == "notification.updated"
assert "_user_id" not in event
@pytest.mark.asyncio
async def test_expire_job_is_locked_set_based_and_commits() -> None:
session = SimpleNamespace(
scalar=AsyncMock(return_value=True),
execute=AsyncMock(
side_effect=[
SimpleNamespace(rowcount=3),
SimpleNamespace(rowcount=2),
]
),
add=Mock(),
commit=AsyncMock(),
)
assert await expire_notifications(session) == (3, 2)
assert session.execute.await_count == 2
personal_sql = str(session.execute.await_args_list[0].args[0])
guest_sql = str(session.execute.await_args_list[1].args[0])
assert "lifecycle_status" in personal_sql and "date_expired" in personal_sql
assert "lifecycle_status" in guest_sql and "date_expired" in guest_sql
session.commit.assert_awaited_once()
@pytest.mark.asyncio
async def test_draft_cleanup_deletes_objects_but_keeps_submitted_object(
monkeypatch: pytest.MonkeyPatch,
) -> None:
abandoned = ClientUploadDraft(
id=uuid.uuid4(),
user_id=uuid.uuid4(),
context_type="notification",
context_id=uuid.uuid4(),
original_file_name="a.pdf",
safe_file_name="a.pdf",
mime_type="application/pdf",
size_bytes=1,
storage_bucket="quarantine",
object_key="q/a",
quarantine_object_key="q/a",
state="draft",
)
submitted = ClientUploadDraft(
id=uuid.uuid4(),
user_id=abandoned.user_id,
context_type="notification",
context_id=abandoned.context_id,
original_file_name="b.pdf",
safe_file_name="b.pdf",
mime_type="application/pdf",
size_bytes=1,
storage_bucket="attachments",
object_key="a/b",
quarantine_object_key=None,
state="submitted",
)
session = SimpleNamespace(
execute=AsyncMock(side_effect=[ScalarRows([abandoned, submitted]), None, None]),
commit=AsyncMock(),
)
class Sessions:
async def __aenter__(self) -> object:
return session
async def __aexit__(self, *_args: object) -> None:
return None
db = SimpleNamespace(sessions=lambda: Sessions())
s3 = SimpleNamespace(delete_quarantine=AsyncMock(), delete=AsyncMock())
monkeypatch.setattr(
"app.workers.load_settings",
AsyncMock(
return_value=SettingsSnapshot(
{"notification.upload_draft.ttl_days": "7"}, "v1"
)
),
)
assert await notification_draft_cleanup_once(db, s3) == 2
s3.delete_quarantine.assert_awaited_once_with("q/a")
s3.delete.assert_not_awaited()
assert session.execute.await_count == 3
session.commit.assert_awaited_once()
VM1_app/codebase/backend/api-backend/tests/unit/test_telemetry.py
+25
View File
@@ -0,0 +1,25 @@
from opentelemetry.sdk.trace import TracerProvider
from app import telemetry
def test_telemetry_is_fail_open_without_endpoint(monkeypatch) -> None:
monkeypatch.delenv("OTEL_EXPORTER_OTLP_ENDPOINT", raising=False)
monkeypatch.setattr(telemetry, "_runtime", None)
assert telemetry.init_telemetry("test-service") is None
def test_structlog_processor_adds_active_trace_context_only() -> None:
provider = TracerProvider()
tracer = provider.get_tracer("test")
event = {"event": "safe", "request_id": "request-1"}
with tracer.start_as_current_span("operation"):
result = telemetry.add_trace_context(None, "info", event)
assert result["event"] == "safe"
assert result["request_id"] == "request-1"
assert len(result["trace_id"]) == 32
assert len(result["span_id"]) == 16
assert set(result) == {"event", "request_id", "trace_id", "span_id"}
VM1_app/codebase/backend/bitrix-local-app/Dockerfile
+14
View File
@@ -0,0 +1,14 @@
FROM python:3.12-slim
ENV PYTHONDONTWRITEBYTECODE=1 PYTHONUNBUFFERED=1
RUN addgroup --system app && adduser --system --ingroup app app
WORKDIR /service
COPY app ./app
COPY alembic ./alembic
COPY alembic.ini pyproject.toml ./
RUN pip install --no-cache-dir .
COPY --chmod=0555 container-entrypoint.sh /usr/local/bin/han-container-entrypoint
USER app
EXPOSE 8080
HEALTHCHECK --interval=30s --timeout=3s CMD python -c "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health/live', timeout=2)"
ENTRYPOINT ["/usr/local/bin/han-container-entrypoint"]
CMD ["uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "8080"]
VM1_app/codebase/backend/bitrix-local-app/alembic.ini
+30
View File
@@ -0,0 +1,30 @@
[alembic]
script_location = alembic
prepend_sys_path = .
sqlalchemy.url = postgresql+asyncpg://unused
[loggers]
keys = root,sqlalchemy,alembic
[handlers]
keys = console
[formatters]
keys = generic
[logger_root]
level = WARN
handlers = console
qualname =
[logger_sqlalchemy]
level = WARN
handlers =
qualname = sqlalchemy.engine
[logger_alembic]
level = INFO
handlers =
qualname = alembic
[handler_console]
class = StreamHandler
args = (sys.stderr,)
level = NOTSET
formatter = generic
[formatter_generic]
format = %(levelname)-5.5s [%(name)s] %(message)s
VM1_app/codebase/backend/bitrix-local-app/alembic/env.py
+42
View File
@@ -0,0 +1,42 @@
import asyncio
import os
from alembic import context
from app.models import Base
from app.postgres import create_postgres_engine
config = context.config
url = os.environ["BITRIX_DATABASE_URL"]
config.set_main_option("sqlalchemy.url", url.replace("%", "%%"))
target_metadata = Base.metadata
def run_offline() -> None:
context.configure(
url=config.get_main_option("sqlalchemy.url"),
target_metadata=target_metadata,
literal_binds=True,
include_schemas=True,
)
with context.begin_transaction():
context.run_migrations()
def do_run(connection) -> None:
context.configure(connection=connection, target_metadata=target_metadata, include_schemas=True)
with context.begin_transaction():
context.run_migrations()
async def run_online() -> None:
engine = create_postgres_engine(url)
async with engine.connect() as connection:
await connection.run_sync(do_run)
await engine.dispose()
if context.is_offline_mode():
run_offline()
else:
asyncio.run(run_online())
VM1_app/codebase/backend/bitrix-local-app/alembic/versions/0001_bitrix_local_schema.py
+20
View File
@@ -0,0 +1,20 @@
"""Create durable OAuth, mapping, inbox, outbox, setup and audit storage."""
from alembic import op
from app.models import Base
revision = "0001_bitrix_local"
down_revision = None
branch_labels = None
depends_on = None
def upgrade() -> None:
bind = op.get_bind()
op.execute("CREATE SCHEMA IF NOT EXISTS bitrix_local")
Base.metadata.create_all(bind=bind, checkfirst=False)
def downgrade() -> None:
Base.metadata.drop_all(bind=op.get_bind(), checkfirst=True)
VM1_app/codebase/backend/bitrix-local-app/app/__init__.py
+1
View File
@@ -0,0 +1 @@
"""HAN Bitrix24 Open Lines adapter."""
VM1_app/codebase/backend/bitrix-local-app/app/main.py
+1333
View File
File diff suppressed because it is too large Load Diff
VM1_app/codebase/backend/bitrix-local-app/app/models.py
+188
View File
@@ -0,0 +1,188 @@
from __future__ import annotations
import uuid
from datetime import UTC, datetime
from sqlalchemy import (
BigInteger,
DateTime,
ForeignKey,
Index,
Integer,
JSON,
String,
Text,
UniqueConstraint,
text,
)
from sqlalchemy.dialects.postgresql import UUID
from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column
SCHEMA = "bitrix_local"
def now() -> datetime:
return datetime.now(UTC)
class Base(DeclarativeBase):
pass
class Common:
id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=now)
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=now, onupdate=now)
record_status: Mapped[str] = mapped_column(String(1), default="A")
status_changed_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
status_change_reason: Mapped[str | None] = mapped_column(String(255))
class PortalInstallation(Common, Base):
__tablename__ = "portal_installations"
__table_args__ = (
Index(
"uq_portal_active_domain",
"domain",
unique=True,
postgresql_where=text("record_status = 'A'"),
),
{"schema": SCHEMA},
)
member_id: Mapped[str] = mapped_column(String(128), unique=True)
domain: Mapped[str] = mapped_column(String(255))
client_endpoint: Mapped[str] = mapped_column(String(1024))
access_ciphertext: Mapped[str] = mapped_column(Text)
access_nonce: Mapped[str] = mapped_column(String(64))
refresh_ciphertext: Mapped[str] = mapped_column(Text)
refresh_nonce: Mapped[str] = mapped_column(String(64))
application_ciphertext: Mapped[str] = mapped_column(Text)
application_nonce: Mapped[str] = mapped_column(String(64))
key_version: Mapped[str] = mapped_column(String(32))
expires_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
scope: Mapped[str | None] = mapped_column(Text)
install_status: Mapped[str] = mapped_column(String(32), default="installed")
setup_status: Mapped[str] = mapped_column(String(32), default="pending")
last_refresh_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
last_error_code: Mapped[str | None] = mapped_column(String(64))
class ConnectorSetup(Common, Base):
__tablename__ = "connector_setup"
__table_args__ = (
UniqueConstraint("portal_id", "connector_id", "line_id"),
Index("ix_setup_retry", "next_retry_at", "attempt_count"),
{"schema": SCHEMA},
)
portal_id: Mapped[uuid.UUID] = mapped_column(
UUID(as_uuid=True), ForeignKey(f"{SCHEMA}.portal_installations.id")
)
connector_id: Mapped[str] = mapped_column(String(64))
line_id: Mapped[str] = mapped_column(String(32))
registered: Mapped[bool] = mapped_column(default=False)
activated: Mapped[bool] = mapped_column(default=False)
bindings_json: Mapped[dict] = mapped_column(JSON, default=dict)
desired_version: Mapped[str] = mapped_column(String(32), default="1")
observed_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
next_retry_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
attempt_count: Mapped[int] = mapped_column(Integer, default=0)
lease_until: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
last_error_code: Mapped[str | None] = mapped_column(String(64))
class DialogSession(Common, Base):
__tablename__ = "dialog_sessions"
__table_args__ = (
Index(
"uq_dialog_active_external_chat",
"external_chat_id",
unique=True,
postgresql_where=text("record_status = 'A'"),
),
Index("ix_dialog_bitrix_chat", "bitrix_chat_id"),
Index("ix_dialog_session", "session_id"),
Index("ix_dialog_status_updated", "status", "updated_at"),
{"schema": SCHEMA},
)
external_chat_id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True))
bitrix_chat_id: Mapped[int | None] = mapped_column(BigInteger)
session_id: Mapped[str | None] = mapped_column(String(255))
portal_id: Mapped[uuid.UUID] = mapped_column(
UUID(as_uuid=True), ForeignKey(f"{SCHEMA}.portal_installations.id")
)
status: Mapped[str] = mapped_column(String(16), default="open")
class InboxEvent(Common, Base):
__tablename__ = "inbox_events"
__table_args__ = (
UniqueConstraint("event_id"),
Index(
"uq_inbox_chat_message",
"external_chat_id",
"bitrix_message_id",
unique=True,
postgresql_where=text("bitrix_message_id IS NOT NULL"),
),
Index("ix_inbox_worker", "status", "next_attempt_at"),
{"schema": SCHEMA},
)
event_id: Mapped[str] = mapped_column(String(255))
event_type: Mapped[str] = mapped_column(String(64))
external_chat_id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True))
bitrix_message_id: Mapped[str | None] = mapped_column(String(255))
payload_fingerprint: Mapped[str] = mapped_column(String(64))
normalized_json: Mapped[dict] = mapped_column(JSON)
status: Mapped[str] = mapped_column(String(32), default="received")
attempt_count: Mapped[int] = mapped_column(Integer, default=0)
next_attempt_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=now)
lease_until: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
api_ack_status: Mapped[str | None] = mapped_column(String(32))
delivery_ack_status: Mapped[str | None] = mapped_column(String(32))
last_error_code: Mapped[str | None] = mapped_column(String(64))
class OutboundMessage(Common, Base):
__tablename__ = "outbound_messages"
__table_args__ = (Index("ix_outbound_worker", "status", "next_attempt_at"), {"schema": SCHEMA})
message_id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), unique=True)
external_chat_id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True))
request_fingerprint: Mapped[str] = mapped_column(String(64))
payload_json: Mapped[dict] = mapped_column(JSON)
status: Mapped[str] = mapped_column(String(32), default="received")
bitrix_message_id: Mapped[str | None] = mapped_column(String(255))
response_json: Mapped[dict | None] = mapped_column(JSON)
attempt_count: Mapped[int] = mapped_column(Integer, default=0)
next_attempt_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=now)
lease_until: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
last_error_code: Mapped[str | None] = mapped_column(String(64))
class DeliveryAckOutbox(Common, Base):
__tablename__ = "delivery_ack_outbox"
__table_args__ = (Index("ix_ack_worker", "status", "next_attempt_at"), {"schema": SCHEMA})
inbox_event_id: Mapped[uuid.UUID] = mapped_column(
UUID(as_uuid=True), ForeignKey(f"{SCHEMA}.inbox_events.id"), unique=True
)
payload_json: Mapped[dict] = mapped_column(JSON)
status: Mapped[str] = mapped_column(String(32), default="pending")
attempt_count: Mapped[int] = mapped_column(Integer, default=0)
next_attempt_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=now)
lease_until: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
last_error_code: Mapped[str | None] = mapped_column(String(64))
class InstallRun(Common, Base):
__tablename__ = "install_runs"
__table_args__ = ({"schema": SCHEMA},)
portal_id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True))
status: Mapped[str] = mapped_column(String(32))
result_json: Mapped[dict] = mapped_column(JSON, default=dict)
class AuditEvent(Common, Base):
__tablename__ = "audit_events"
__table_args__ = (Index("ix_audit_created", "created_at"), {"schema": SCHEMA})
event_type: Mapped[str] = mapped_column(String(128))
actor_type: Mapped[str] = mapped_column(String(32), default="system")
safe_details: Mapped[dict] = mapped_column(JSON, default=dict)
VM1_app/codebase/backend/bitrix-local-app/app/postgres.py
+25
View File
@@ -0,0 +1,25 @@
from __future__ import annotations
from typing import Any
import asyncpg
from sqlalchemy.ext.asyncio import AsyncEngine, create_async_engine
def asyncpg_dsn(url: str) -> str:
if url.startswith("postgresql+asyncpg://"):
return url.replace("postgresql+asyncpg://", "postgresql://", 1)
return url
def create_postgres_engine(url: str, **engine_options: Any) -> AsyncEngine:
dsn = asyncpg_dsn(url)
async def connect():
return await asyncpg.connect(dsn=dsn)
return create_async_engine(
"postgresql+asyncpg://",
async_creator=connect,
**engine_options,
)
VM1_app/codebase/backend/bitrix-local-app/container-entrypoint.sh
+23
View File
@@ -0,0 +1,23 @@
#!/bin/sh
set -eu
for name in ${HAN_SECRET_VARS:-}; do
case "$name" in
""|[0-9]*|*[!A-Z0-9_]*)
echo "container secrets: invalid variable name" >&2
exit 64
;;
*) ;;
esac
eval "file=\${${name}_FILE:-}"
if [ -z "$file" ] || [ ! -r "$file" ]; then
echo "container secrets: missing file for $name" >&2
exit 66
fi
value=$(cat "$file")
export "$name=$value"
unset "${name}_FILE"
done
unset HAN_SECRET_VARS
exec "$@"
VM1_app/codebase/backend/bitrix-local-app/openapi.yaml
+104
View File
@@ -0,0 +1,104 @@
openapi: 3.1.0
info: {title: HAN Bitrix24 Local App, version: 1.0.0}
paths:
/bitrix/handler:
get: {responses: {"200": {description: Callback probe}}}
post:
requestBody:
required: true
content:
application/json: {schema: {type: object}}
application/x-www-form-urlencoded: {schema: {type: object}}
multipart/form-data: {schema: {type: object}}
responses:
"200": {description: Duplicate or ignored callback}
"202": {description: Durably accepted callback}
"400": {description: Invalid callback}
"403": {description: Invalid application token}
/bitrix/install:
get: {responses: {"200": {description: Install probe}}}
post:
responses:
"200": {description: OAuth saved and setup attempted}
"400": {description: Invalid install callback}
/bitrix/placement:
get: {responses: {"200": {description: Connector placement HTML}}}
/health/live:
get: {responses: {"200": {description: Live}}}
/health/ready:
get: {responses: {"200": {description: Ready}, "503": {description: Not ready}}}
/internal/openlines/v1/messages:
post:
security: [{BearerAuth: []}]
parameters:
- {name: Idempotency-Key, in: header, required: true, schema: {type: string}}
- {$ref: "#/components/parameters/RequestId"}
requestBody:
required: true
content:
application/json: {schema: {$ref: "#/components/schemas/OutboundMessage"}}
responses:
"201": {description: Delivered}
"200": {description: Idempotent duplicate}
"400": {description: Invalid request}
"401": {description: Unauthorized}
"409": {description: Idempotency key reused}
"503": {description: Dependency unavailable or ambiguous delivery}
/internal/openlines/v1/dialogs/{external_chat_id}:
get:
security: [{BearerAuth: []}]
parameters:
- {name: external_chat_id, in: path, required: true, schema: {type: string, format: uuid}}
responses:
"200": {description: Active dialog mapping}
"404": {description: Mapping not found}
/internal/openlines/v1/status:
get:
security: [{BearerAuth: []}]
responses: {"200": {description: Safe adapter status}}
/internal/openlines/v1/setup/retry:
post:
security: [{BearerAuth: []}]
responses: {"200": {description: Idempotent setup reconcile result}}
components:
securitySchemes:
BearerAuth: {type: http, scheme: bearer}
parameters:
RequestId: {name: X-Request-ID, in: header, required: false, schema: {type: string}}
schemas:
OutboundFile:
type: object
additionalProperties: false
required: [attachment_id, name, mime_type, size_bytes, download_url]
properties:
attachment_id: {type: string, format: uuid}
name: {type: string, maxLength: 255}
mime_type: {type: string, maxLength: 255}
size_bytes: {type: integer, minimum: 1, maximum: 5242880}
download_url: {type: string, maxLength: 4096}
OutboundMessage:
type: object
additionalProperties: false
required: [message_id, external_chat_id, occurred_at, user, message]
properties:
message_id: {type: string, format: uuid}
external_chat_id: {type: string, format: uuid}
occurred_at: {type: string, format: date-time}
user:
type: object
additionalProperties: false
required: [id, display_name]
properties:
id: {type: string, format: uuid}
display_name: {type: string, maxLength: 255}
message:
type: object
additionalProperties: false
required: [content_kind, text, files]
properties:
content_kind: {type: string, enum: [text, file]}
text: {type: string, maxLength: 10000}
files:
type: array
maxItems: 1
items: {$ref: "#/components/schemas/OutboundFile"}
VM1_app/codebase/backend/bitrix-local-app/pyproject.toml
+33
View File
@@ -0,0 +1,33 @@
[project]
name = "han-bitrix-local-app"
version = "0.1.0"
requires-python = ">=3.12"
dependencies = [
"alembic>=1.16,<2",
"asyncpg>=0.30,<1",
"cryptography>=45,<46",
"fastapi>=0.116,<1",
"httpx>=0.28,<1",
"pydantic-settings>=2.10,<3",
"python-multipart>=0.0.20,<1",
"sqlalchemy[asyncio]>=2.0.41,<3",
"uvicorn[standard]>=0.35,<1",
]
[project.optional-dependencies]
dev = ["pytest>=8.4,<9", "pytest-asyncio>=1,<2", "ruff>=0.12,<1"]
[build-system]
requires = ["hatchling"]
build-backend = "hatchling.build"
[tool.hatch.build.targets.wheel]
packages = ["app"]
[tool.pytest.ini_options]
asyncio_mode = "auto"
testpaths = ["tests"]
[tool.ruff]
target-version = "py312"
line-length = 100
VM1_app/codebase/backend/bitrix-local-app/tests/test_core.py
+280
View File
@@ -0,0 +1,280 @@
import base64
import os
import uuid
from types import SimpleNamespace
from urllib.parse import parse_qs
os.environ.setdefault("BITRIX_DATABASE_URL", "postgresql://unused/unused")
os.environ.setdefault("BITRIX_CLIENT_ID", "client")
os.environ.setdefault("BITRIX_CLIENT_SECRET", "secret")
os.environ.setdefault("BITRIX_APPLICATION_TOKEN", "application-token")
os.environ.setdefault("BITRIX_INTERNAL_API_TOKEN", "internal-token-32-characters-long")
os.environ.setdefault("BITRIX_API_FORWARD_URL", "http://api/internal/openlines/v1/inbox")
os.environ.setdefault("BITRIX_API_FORWARD_TOKEN", "forward-token-32-characters-long")
os.environ.setdefault(
"BITRIX_TOKEN_ENCRYPTION_KEY",
base64.urlsafe_b64encode(b"k" * 32).decode().rstrip("="),
)
import httpx
import pytest
from app.main import (
BitrixClient,
TokenCipher,
canonical_fingerprint,
normalize_event,
resolve_inbound_file_urls,
retry_delay,
safely_retryable,
validate_portal,
)
def test_token_cipher_binds_aad():
cipher = TokenCipher(os.environ["BITRIX_TOKEN_ENCRYPTION_KEY"], "v1")
ciphertext, nonce = cipher.encrypt("secret", "member", "han0107.bitrix24.ru", "access")
assert cipher.decrypt(ciphertext, nonce, "member", "han0107.bitrix24.ru", "access") == "secret"
with pytest.raises(Exception):
cipher.decrypt(ciphertext, nonce, "other", "han0107.bitrix24.ru", "access")
def test_normalize_message_and_finish():
external = str(uuid.uuid4())
message = normalize_event(
{
"event": "ONIMCONNECTORMESSAGEADD",
"data": {
"MESSAGES": [
{
"im": {"chat_id": 1807, "message_id": 86497},
"chat": {"id": external},
"message": {"text": "Ответ", "files": []},
}
]
},
}
)
assert message["event_type"] == "message.new"
assert message["external_chat_id"] == external
assert message["bitrix_message_id"] == "86497"
assert message["message"]["text"] == "Ответ"
closed = normalize_event(
{"event": "ONIMCONNECTORDIALOGFINISH", "data": {"external_chat_id": external}}
)
assert closed["event_type"] == "dialog.closed"
def test_normalize_message_removes_bitrix_sender_prefix():
external = str(uuid.uuid4())
message = normalize_event(
{
"event": "ONIMCONNECTORMESSAGEADD",
"data": {
"MESSAGES": [
{
"im": {"message_id": 86498},
"chat": {"id": external},
"message": {
"text": "[b]Антон Пичугин:[/b] [br]опять ты?",
"files": [],
},
}
]
},
}
)
assert message["message"]["text"] == "опять ты?"
def test_normalize_bitrix_file_uses_download_url_and_infers_mime_type():
external = str(uuid.uuid4())
message = normalize_event(
{
"event": "ONIMCONNECTORMESSAGEADD",
"data": {
"MESSAGES": [
{
"im": {"chat_id": 1807, "message_id": 86498},
"chat": {"id": external},
"message": {
"text": "",
"files": [
{
"name": "image.png",
"type": "image",
"size": 941380,
"urlDownload": "https://portal.bitrix24.ru/download/file.png",
}
],
},
}
]
},
}
)
assert message["message"]["files"] == [
{
"name": "image.png",
"mime_type": "image/png",
"size_bytes": 941380,
"download_url": "https://portal.bitrix24.ru/download/file.png",
}
]
def test_normalize_bitrix_file_uses_open_lines_download_link_and_mime():
external = str(uuid.uuid4())
message = normalize_event(
{
"event": "ONIMCONNECTORMESSAGEADD",
"data": {
"MESSAGES": [
{
"im": {"chat_id": 1807, "message_id": 86499},
"chat": {"id": external},
"message": {
"text": "",
"files": [
{
"name": "diploma.jpg",
"type": "image",
"mime": "image/jpeg",
"size": 236934,
"downloadLink": (
"https://portal.bitrix24.ru/download/diploma.jpg"
),
"link": "https://portal.bitrix24.ru/view/diploma.jpg",
"width": 1200,
"height": 800,
}
],
},
}
]
},
}
)
assert message["message"]["files"] == [
{
"name": "diploma.jpg",
"mime_type": "image/jpeg",
"size_bytes": 236934,
"download_url": "https://portal.bitrix24.ru/download/diploma.jpg",
}
]
@pytest.mark.asyncio
async def test_resolve_inbound_file_url_from_bitrix_file_id(monkeypatch):
class SessionContext:
async def __aenter__(self):
return object()
async def __aexit__(self, *_args):
return None
class Bitrix:
async def call(self, portal, method, params):
assert portal == "portal"
assert method == "im.v2.File.download"
assert params == {"id": "5155"}
return {"downloadUrl": "https://portal.bitrix24.ru/download/file.png"}
async def fake_active_portal(_session):
return "portal"
monkeypatch.setattr("app.main.active_portal", fake_active_portal)
app = SimpleNamespace(
state=SimpleNamespace(sessions=lambda: SessionContext(), bitrix=Bitrix())
)
payload = {
"message": {
"files": [
{
"name": "image.png",
"mime_type": "image/png",
"size_bytes": 941380,
"download_url": "",
"_bitrix_file_id": "5155",
}
]
}
}
await resolve_inbound_file_urls(app, payload)
assert payload["message"]["files"] == [
{
"name": "image.png",
"mime_type": "image/png",
"size_bytes": 941380,
"download_url": "https://portal.bitrix24.ru/download/file.png",
}
]
def test_fingerprint_ignores_signed_query_and_portal_validation():
payload = {"message": {"files": [{"download_url": "https://s3/object?sig=one"}]}}
other = {"message": {"files": [{"download_url": "https://s3/object?sig=two"}]}}
assert canonical_fingerprint(payload) == canonical_fingerprint(other)
validate_portal(
"han0107.bitrix24.ru",
"https://han0107.bitrix24.ru/rest/",
"han0107.bitrix24.ru",
)
with pytest.raises(ValueError):
validate_portal("evil.example", "https://evil.example/rest/", "han0107.bitrix24.ru")
assert 0 <= retry_delay(4, 300) <= 8
def test_network_timeouts_are_retryable():
assert safely_retryable(TimeoutError("Delivery operation timed out"))
assert safely_retryable(httpx.ReadTimeout("Bitrix response timed out"))
assert safely_retryable(httpx.ConnectTimeout("Bitrix connection timed out"))
@pytest.mark.asyncio
async def test_bitrix_call_refreshes_and_retries_once_after_401():
auth_values: list[str] = []
async def handler(request: httpx.Request) -> httpx.Response:
auth = parse_qs(request.content.decode())["auth"][0]
auth_values.append(auth)
if auth == "old-access":
return httpx.Response(401, request=request)
return httpx.Response(200, json={"result": {"ok": True}}, request=request)
class Cipher:
@staticmethod
def decrypt(ciphertext, *_):
return ciphertext
portal = SimpleNamespace(
access_ciphertext="old-access",
access_nonce="nonce",
member_id="member",
domain="han0107.bitrix24.ru",
expires_at=None,
)
settings = SimpleNamespace(bitrix_http_max_concurrency=2)
async with httpx.AsyncClient(transport=httpx.MockTransport(handler)) as http:
client = BitrixClient(http, settings, Cipher(), sessions=None)
refresh_calls: list[bool] = []
async def ensure_fresh(_, *, force=False, stale_access_ciphertext=None):
refresh_calls.append(force)
if force:
assert stale_access_ciphertext == "old-access"
portal.access_ciphertext = "new-access"
client.ensure_fresh = ensure_fresh
result = await client.call(portal, "imconnector.send.messages", {"MESSAGE": "test"})
assert result == {"ok": True}
assert auth_values == ["old-access", "new-access"]
assert refresh_calls == [False, True]
VM1_app/codebase/backend/bitrix-sync/Dockerfile
+14
View File
@@ -0,0 +1,14 @@
FROM python:3.12-slim
ENV PYTHONDONTWRITEBYTECODE=1 PYTHONUNBUFFERED=1
RUN addgroup --system app && adduser --system --ingroup app app
WORKDIR /service
COPY app ./app
COPY alembic ./alembic
COPY alembic.ini pyproject.toml ./
RUN pip install --no-cache-dir .
COPY --chmod=0555 container-entrypoint.sh /usr/local/bin/han-container-entrypoint
USER app
EXPOSE 8080
HEALTHCHECK --interval=30s --timeout=3s CMD python -c "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health/live', timeout=2)"
ENTRYPOINT ["/usr/local/bin/han-container-entrypoint"]
CMD ["uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "8080"]
VM1_app/codebase/backend/bitrix-sync/alembic.ini
+30
View File
@@ -0,0 +1,30 @@
[alembic]
script_location = alembic
prepend_sys_path = .
sqlalchemy.url = postgresql+asyncpg://unused
[loggers]
keys = root,sqlalchemy,alembic
[handlers]
keys = console
[formatters]
keys = generic
[logger_root]
level = WARN
handlers = console
qualname =
[logger_sqlalchemy]
level = WARN
handlers =
qualname = sqlalchemy.engine
[logger_alembic]
level = INFO
handlers =
qualname = alembic
[handler_console]
class = StreamHandler
args = (sys.stderr,)
level = NOTSET
formatter = generic
[formatter_generic]
format = %(levelname)-5.5s [%(name)s] %(message)s
VM1_app/codebase/backend/bitrix-sync/alembic/env.py
+38
View File
@@ -0,0 +1,38 @@
import asyncio
import os
from alembic import context
from app.postgres import create_postgres_engine
config = context.config
database_url = os.environ["BITRIX_SYNC_DATABASE_URL"]
config.set_main_option(
"sqlalchemy.url",
database_url.replace("%", "%%"),
)
target_metadata = None
def run_offline() -> None:
context.configure(url=config.get_main_option("sqlalchemy.url"), literal_binds=True)
with context.begin_transaction():
context.run_migrations()
async def run_online() -> None:
engine = create_postgres_engine(database_url)
async with engine.connect() as connection:
await connection.run_sync(do_run)
await engine.dispose()
def do_run(connection) -> None:
context.configure(connection=connection)
with context.begin_transaction():
context.run_migrations()
if context.is_offline_mode():
run_offline()
else:
asyncio.run(run_online())
VM1_app/codebase/backend/bitrix-sync/alembic/versions/0001_connectivity_stub_baseline.py
+15
View File
@@ -0,0 +1,15 @@
"""Establish the bitrix-sync connectivity-stub migration baseline."""
revision = "0001_sync_baseline"
down_revision = None
branch_labels = None
depends_on = None
def upgrade() -> None:
# The connectivity stub deliberately owns no runtime tables.
pass
def downgrade() -> None:
pass
VM1_app/codebase/backend/bitrix-sync/app/__init__.py
+1
View File
@@ -0,0 +1 @@
"""HAN bitrix-sync DB connectivity stub."""
VM1_app/codebase/backend/bitrix-sync/app/main.py
+315
View File
@@ -0,0 +1,315 @@
from __future__ import annotations
import asyncio
import hmac
import random
import time
import uuid
from contextlib import asynccontextmanager, suppress
from dataclasses import dataclass
from datetime import UTC, datetime, timedelta
from typing import Annotated, Protocol
import uvicorn
from fastapi import FastAPI, Header, HTTPException, Request
from fastapi.responses import JSONResponse
from pydantic import Field, model_validator
from pydantic_settings import BaseSettings, SettingsConfigDict
from sqlalchemy import text
from sqlalchemy.ext.asyncio import AsyncEngine
from app.postgres import create_postgres_engine
def utcnow() -> datetime:
return datetime.now(UTC)
class Settings(BaseSettings):
model_config = SettingsConfigDict(extra="ignore")
app_env: str = "production-like"
bitrix_sync_enabled: bool = True
bitrix_sync_database_url: str | None = None
bitrix_sync_service_token: str = Field(min_length=16)
bitrix_sync_db_check_interval_sec: float = Field(default=60, ge=0.05)
bitrix_sync_db_check_timeout_sec: float = Field(default=5, ge=0.05)
bitrix_sync_db_check_jitter_ratio: float = Field(default=0.1, ge=0, le=0.5)
bitrix_sync_db_retry_base_sec: float = Field(default=5, ge=0.05)
bitrix_sync_db_retry_max_sec: float = Field(default=60, ge=0.05)
bitrix_sync_ready_max_staleness_sec: float = Field(default=150, ge=1)
bitrix_sync_db_pool_size: int = Field(default=2, ge=1, le=5)
bitrix_sync_db_pool_recycle_sec: int = Field(default=300, ge=30)
@model_validator(mode="after")
def validate_enabled(self) -> Settings:
if self.bitrix_sync_enabled and not self.bitrix_sync_database_url:
raise ValueError("BITRIX_SYNC_DATABASE_URL is required when sync is enabled")
return self
@dataclass
class Snapshot:
state: str
started_at: datetime
last_started_at: datetime | None = None
last_finished_at: datetime | None = None
last_success_at: datetime | None = None
success: bool | None = None
duration_ms: int | None = None
error_code: str | None = None
consecutive_failures: int = 0
next_check_at: datetime | None = None
worker_running: bool = False
class Probe(Protocol):
async def check(self) -> None: ...
async def close(self) -> None: ...
class DatabaseProbe:
def __init__(self, engine: AsyncEngine) -> None:
self.engine = engine
async def check(self) -> None:
async with self.engine.connect() as connection:
result = await connection.scalar(text("SELECT 1"))
if result != 1:
raise RuntimeError("unexpected_result")
async def close(self) -> None:
await self.engine.dispose()
def classify_error(exc: Exception) -> str:
name = type(exc).__name__.lower()
text_value = str(exc).lower()
if isinstance(exc, TimeoutError):
return "db_query_timeout"
if "auth" in name or "password" in text_value:
return "db_auth_failed"
if "ssl" in name or "tls" in text_value or "certificate" in text_value:
return "db_tls_failed"
if str(exc) == "unexpected_result":
return "unexpected_result"
return "db_unavailable"
def jsonable(value):
if isinstance(value, datetime):
return value.isoformat().replace("+00:00", "Z")
return value
def create_app(settings: Settings | None = None, probe: Probe | None = None) -> FastAPI:
cfg = settings or Settings()
@asynccontextmanager
async def lifespan(app: FastAPI):
app.state.snapshot = Snapshot(
state="disabled" if not cfg.bitrix_sync_enabled else "starting",
started_at=utcnow(),
)
app.state.stop = asyncio.Event()
app.state.probe = probe
app.state.loop_task = None
if cfg.bitrix_sync_enabled:
if app.state.probe is None:
engine = create_postgres_engine(
cfg.bitrix_sync_database_url or "",
pool_size=cfg.bitrix_sync_db_pool_size,
max_overflow=0,
pool_pre_ping=True,
pool_recycle=cfg.bitrix_sync_db_pool_recycle_sec,
server_settings={"application_name": "han-bitrix-sync"},
)
app.state.probe = DatabaseProbe(engine)
await run_probe(app)
app.state.snapshot.worker_running = True
app.state.loop_task = asyncio.create_task(
periodic_loop(app), name="db-connectivity-probe"
)
yield
app.state.snapshot.state = "stopping"
app.state.stop.set()
if app.state.loop_task:
app.state.loop_task.cancel()
with suppress(asyncio.CancelledError):
await app.state.loop_task
if app.state.probe:
await app.state.probe.close()
app = FastAPI(
title="HAN Bitrix Sync Connectivity Stub",
version="1.0.0",
lifespan=lifespan,
docs_url=None if cfg.app_env != "test" else "/docs",
)
app.state.settings = cfg
@app.middleware("http")
async def request_id(request: Request, call_next):
request.state.request_id = request.headers.get("X-Request-ID") or str(uuid.uuid4())
response = await call_next(request)
response.headers["X-Request-ID"] = request.state.request_id
return response
@app.exception_handler(HTTPException)
async def http_error(_: Request, exc: HTTPException):
return JSONResponse(status_code=exc.status_code, content=exc.detail)
@app.get("/health/live")
async def live():
return {"status": "live"}
@app.get("/health/ready")
async def ready(request: Request):
snapshot: Snapshot = request.app.state.snapshot
if not cfg.bitrix_sync_enabled:
return JSONResponse({"status": "not_ready", "reason": "sync_disabled"}, status_code=503)
age = (
(utcnow() - snapshot.last_success_at).total_seconds()
if snapshot.last_success_at
else None
)
max_age = max(
cfg.bitrix_sync_ready_max_staleness_sec,
2 * cfg.bitrix_sync_db_check_interval_sec * (1 + cfg.bitrix_sync_db_check_jitter_ratio),
)
is_ready = (
snapshot.state == "healthy"
and snapshot.worker_running
and age is not None
and age <= max_age
)
if is_ready:
return {
"status": "ready",
"mode": "db_connectivity_stub",
"database": {
"status": "ok",
"last_success_at": jsonable(snapshot.last_success_at),
"age_seconds": round(age or 0, 3),
},
}
reason = "worker_not_running" if not snapshot.worker_running else "database_unavailable"
return JSONResponse(
{
"status": "not_ready",
"reason": reason,
"database": {
"status": "down",
"last_success_at": jsonable(snapshot.last_success_at),
"consecutive_failures": snapshot.consecutive_failures,
},
},
status_code=503,
)
@app.get("/internal/sync/v1/status")
async def status(
request: Request,
authorization: Annotated[str | None, Header()] = None,
):
candidate = (
authorization[7:] if authorization and authorization.startswith("Bearer ") else ""
)
if not hmac.compare_digest(candidate, cfg.bitrix_sync_service_token):
raise HTTPException(
401,
{
"error": {
"code": "service_unauthorized",
"message": "Service authentication failed",
"request_id": request.state.request_id,
"details": {},
}
},
)
snapshot: Snapshot = request.app.state.snapshot
last_check = None
if snapshot.last_started_at:
last_check = {
"started_at": jsonable(snapshot.last_started_at),
"finished_at": jsonable(snapshot.last_finished_at),
"success": snapshot.success,
"duration_ms": snapshot.duration_ms,
"error_code": snapshot.error_code,
}
next_in = (
max(0, (snapshot.next_check_at - utcnow()).total_seconds())
if snapshot.next_check_at
else None
)
return {
"service": "bitrix-sync",
"enabled": cfg.bitrix_sync_enabled,
"mode": "db_connectivity_stub",
"crm_sync_implemented": False,
"state": snapshot.state,
"started_at": jsonable(snapshot.started_at),
"last_check": last_check,
"last_success_at": jsonable(snapshot.last_success_at),
"consecutive_failures": snapshot.consecutive_failures,
"next_check_in_seconds": round(next_in, 3) if next_in is not None else None,
}
return app
async def run_probe(app: FastAPI) -> None:
cfg: Settings = app.state.settings
snapshot: Snapshot = app.state.snapshot
snapshot.last_started_at = utcnow()
started = time.monotonic()
try:
async with asyncio.timeout(cfg.bitrix_sync_db_check_timeout_sec):
await app.state.probe.check()
except Exception as exc:
snapshot.success = False
snapshot.error_code = classify_error(exc)
snapshot.consecutive_failures += 1
snapshot.state = "degraded"
else:
snapshot.success = True
snapshot.error_code = None
snapshot.consecutive_failures = 0
snapshot.last_success_at = utcnow()
snapshot.state = "healthy"
finally:
snapshot.last_finished_at = utcnow()
snapshot.duration_ms = round((time.monotonic() - started) * 1000)
async def periodic_loop(app: FastAPI) -> None:
cfg: Settings = app.state.settings
snapshot: Snapshot = app.state.snapshot
try:
while not app.state.stop.is_set():
if snapshot.consecutive_failures:
cap = min(
cfg.bitrix_sync_db_retry_base_sec * 2 ** (snapshot.consecutive_failures - 1),
cfg.bitrix_sync_db_retry_max_sec,
)
delay = random.uniform(0, cap)
else:
jitter = (
cfg.bitrix_sync_db_check_interval_sec * cfg.bitrix_sync_db_check_jitter_ratio
)
delay = cfg.bitrix_sync_db_check_interval_sec + random.uniform(-jitter, jitter)
snapshot.next_check_at = utcnow() + timedelta(seconds=delay)
try:
await asyncio.wait_for(app.state.stop.wait(), timeout=delay)
break
except TimeoutError:
await run_probe(app)
finally:
snapshot.worker_running = False
_settings = Settings()
app = create_app(_settings)
def run() -> None:
uvicorn.run("app.main:app", host="0.0.0.0", port=8080)
VM1_app/codebase/backend/bitrix-sync/app/postgres.py
+30
View File
@@ -0,0 +1,30 @@
from __future__ import annotations
from typing import Any
import asyncpg
from sqlalchemy.ext.asyncio import AsyncEngine, create_async_engine
def asyncpg_dsn(url: str) -> str:
if url.startswith("postgresql+asyncpg://"):
return url.replace("postgresql+asyncpg://", "postgresql://", 1)
return url
def create_postgres_engine(
url: str,
*,
server_settings: dict[str, str] | None = None,
**engine_options: Any,
) -> AsyncEngine:
dsn = asyncpg_dsn(url)
async def connect():
return await asyncpg.connect(dsn=dsn, server_settings=server_settings)
return create_async_engine(
"postgresql+asyncpg://",
async_creator=connect,
**engine_options,
)
VM1_app/codebase/backend/bitrix-sync/container-entrypoint.sh
+23
View File
@@ -0,0 +1,23 @@
#!/bin/sh
set -eu
for name in ${HAN_SECRET_VARS:-}; do
case "$name" in
""|[0-9]*|*[!A-Z0-9_]*)
echo "container secrets: invalid variable name" >&2
exit 64
;;
*) ;;
esac
eval "file=\${${name}_FILE:-}"
if [ -z "$file" ] || [ ! -r "$file" ]; then
echo "container secrets: missing file for $name" >&2
exit 66
fi
value=$(cat "$file")
export "$name=$value"
unset "${name}_FILE"
done
unset HAN_SECRET_VARS
exec "$@"
VM1_app/codebase/backend/bitrix-sync/openapi.yaml
+46
View File
@@ -0,0 +1,46 @@
openapi: 3.1.0
info: {title: HAN Bitrix Sync Connectivity Stub, version: 1.0.0}
paths:
/health/live:
get:
responses:
"200":
description: Process is live
content: {application/json: {schema: {$ref: "#/components/schemas/Live"}}}
/health/ready:
get:
responses:
"200": {description: Latest PostgreSQL probe is fresh and successful}
"503": {description: Disabled, stale, or database unavailable}
/internal/sync/v1/status:
get:
security: [{BearerAuth: []}]
parameters:
- {name: X-Request-ID, in: header, required: false, schema: {type: string}}
responses:
"200":
description: Connectivity-loop status
content: {application/json: {schema: {$ref: "#/components/schemas/Status"}}}
"401": {description: Service authentication failed}
components:
securitySchemes:
BearerAuth: {type: http, scheme: bearer}
schemas:
Live:
type: object
required: [status]
properties: {status: {const: live}}
Status:
type: object
required: [service, enabled, mode, crm_sync_implemented, state, started_at]
properties:
service: {const: bitrix-sync}
enabled: {type: boolean}
mode: {const: db_connectivity_stub}
crm_sync_implemented: {const: false}
state: {type: string, enum: [starting, disabled, healthy, degraded, stopping]}
started_at: {type: string, format: date-time}
last_check: {type: [object, "null"]}
last_success_at: {type: [string, "null"], format: date-time}
consecutive_failures: {type: integer, minimum: 0}
next_check_in_seconds: {type: [number, "null"]}
VM1_app/codebase/backend/bitrix-sync/pyproject.toml
+30
View File
@@ -0,0 +1,30 @@
[project]
name = "han-bitrix-sync"
version = "0.1.0"
requires-python = ">=3.12"
dependencies = [
"alembic>=1.16,<2",
"asyncpg>=0.30,<1",
"fastapi>=0.116,<1",
"pydantic-settings>=2.10,<3",
"sqlalchemy[asyncio]>=2.0.41,<3",
"uvicorn[standard]>=0.35,<1",
]
[project.optional-dependencies]
dev = ["httpx>=0.28,<1", "pytest>=8.4,<9", "pytest-asyncio>=1,<2", "ruff>=0.12,<1"]
[build-system]
requires = ["hatchling"]
build-backend = "hatchling.build"
[tool.hatch.build.targets.wheel]
packages = ["app"]
[tool.pytest.ini_options]
asyncio_mode = "auto"
testpaths = ["tests"]
[tool.ruff]
target-version = "py312"
line-length = 100
VM1_app/codebase/backend/bitrix-sync/tests/test_service.py
+65
View File
@@ -0,0 +1,65 @@
import os
os.environ.setdefault("BITRIX_SYNC_ENABLED", "false")
os.environ.setdefault("BITRIX_SYNC_SERVICE_TOKEN", "test-sync-token-32-characters")
import httpx
import pytest
from app.main import Settings, create_app
class Probe:
def __init__(self, fail=False):
self.fail = fail
self.calls = 0
async def check(self):
self.calls += 1
if self.fail:
raise OSError("down")
async def close(self):
pass
@pytest.mark.asyncio
async def test_disabled_semantics():
settings = Settings(
bitrix_sync_enabled=False,
bitrix_sync_service_token="test-sync-token-32-characters",
)
app = create_app(settings)
async with app.router.lifespan_context(app):
async with httpx.AsyncClient(
transport=httpx.ASGITransport(app=app), base_url="http://test"
) as client:
assert (await client.get("/health/live")).status_code == 200
ready = await client.get("/health/ready")
assert ready.status_code == 503
assert ready.json()["reason"] == "sync_disabled"
status = await client.get(
"/internal/sync/v1/status",
headers={"Authorization": "Bearer test-sync-token-32-characters"},
)
assert status.json()["state"] == "disabled"
assert status.json()["crm_sync_implemented"] is False
@pytest.mark.asyncio
async def test_initial_probe_and_auth():
probe = Probe()
settings = Settings(
bitrix_sync_enabled=True,
bitrix_sync_database_url="postgresql://unused/unused",
bitrix_sync_service_token="test-sync-token-32-characters",
bitrix_sync_db_check_interval_sec=60,
)
app = create_app(settings, probe)
async with app.router.lifespan_context(app):
async with httpx.AsyncClient(
transport=httpx.ASGITransport(app=app), base_url="http://test"
) as client:
assert probe.calls == 1
assert (await client.get("/health/ready")).status_code == 200
assert (await client.get("/internal/sync/v1/status")).status_code == 401
VM1_app/codebase/backend/deployment/DEPLOYMENT_GUIDE.ru.md
+826
View File
@@ -0,0 +1,826 @@
# Подробная инструкция по развертыванию и запуску HAN Chat
Эта инструкция описывает первый запуск **текущего legacy/stub проекта ВМ1** на одной виртуальной
машине с Ubuntu 24.04. Она не разворачивает target ВМ2 Processing и не подтверждает production-готовность Message Safety v2. Все команды предполагают, что проект расположен в
`/opt/han-chat/backend`, а команды Docker Compose выполняются из этого каталога.
PostgreSQL и Selectel S3 не запускаются в Docker Compose: их необходимо создать
заранее как внешние управляемые сервисы. Из интернета должны быть доступны только
порты 80 и 443 виртуальной машины.
## 1. Что потребуется до начала работы
Подготовьте:
1. Виртуальную машину с Ubuntu 24.04 и минимум 4 vCPU, 8 ГБ RAM и 40 ГБ диска.
2. SSH-доступ к VM пользователем с правом `sudo`.
3. Домен, например `chat.example.ru`, и возможность изменить его DNS.
4. Управляемый PostgreSQL, доступный VM по приватной сети.
5. Три приватных бакета Selectel S3.
6. Учетные данные приложения Bitrix24.
7. При необходимости — удаленный OTLP-бэкенд для телеметрии.
8. Локальную копию каталога `HAN_chat_specification/codebase/backend` либо URL
Git-репозитория, из которого его можно получить.
Для первого тестового запуска допустимы mock OTP, заглушка Message Safety и
заглушка bitrix-sync. Они не являются полноценными production-реализациями.
Целевой cutover выполняется по `modules/module-10-deployment-runbook.md`: самостоятельная ВМ2, root Compose/systemd unit, собственный nginx с public exact CRM webhook `80/443` и private Message Safety listener `8443`, раздельные TLS-контуры, secrets/IAM, egress allow-list и local OTEL Collector. Не переносите команды этого single-VM guide на ВМ2 без VM2-specific manifests.
## 2. Первичный вход на VM
Подключитесь к созданной VM облачным пользователем:
```sh
ssh <cloud-user>@<VM_IP>
```
Проверьте версию ОС:
```sh
cat /etc/os-release
```
Должна использоваться Ubuntu 24.04 или более новая версия.
## 3. Передача и запуск скрипта настройки VM
Сначала передайте на VM только подготовительный скрипт. Например, с локального
компьютера:
```sh
scp deployment/scripts/setup-vm.sh <cloud-user>@<VM_IP>:/tmp/setup-vm.sh
```
На VM выполните:
```sh
chmod +x /tmp/setup-vm.sh
sudo /tmp/setup-vm.sh
```
Скрипт:
- обновит Ubuntu и установит базовые пакеты;
- создаст пользователя `deploy`;
- установит Docker Engine и Docker Compose;
- настроит UFW, fail2ban и цепочку `DOCKER-USER`;
- откроет только SSH, HTTP и HTTPS;
- создаст `/opt/han-chat/backend`;
- создаст swap;
- включит автоматические обновления безопасности;
- отключит парольный SSH-вход и X11 forwarding;
- заблокирует локальные пароли `root` и `deploy` после проверки SSH-ключей.
Если `authorized_keys` пользователя `deploy` отсутствует, скрипт остановится до
блокировки паролей. `HARDEN_SSH=true` дополнительно запрещает прямой вход
пользователем `root` и SSH TCP forwarding; включайте этот режим только после
проверки входа пользователем `deploy` по ключу в отдельной сессии.
Если SSH работает на нестандартном порту или имя внешнего интерфейса известно
заранее, передайте параметры:
```sh
sudo SSH_PORT=2222 EXTERNAL_IF=ens3 /tmp/setup-vm.sh
```
После завершения выйдите из SSH-сессии: членство `deploy` в группе `docker`
начинает действовать только после нового входа.
```sh
exit
ssh deploy@<VM_IP>
docker version
docker compose version
```
## 4. Копирование проекта на VM
### Вариант A — через Git
Это предпочтительный вариант: Git применит правило LF для shell-скриптов.
```sh
git clone <URL_РЕПОЗИТОРИЯ> /tmp/han-chat-source
cp -a /tmp/han-chat-source/HAN_chat_specification/codebase/backend/. \
/opt/han-chat/backend/
cd /opt/han-chat/backend
```
Если `HAN_chat_specification` является корнем репозитория:
```sh
cp -a /tmp/han-chat-source/codebase/backend/. /opt/han-chat/backend/
```
### Вариант B — архивом с локального компьютера
Создайте архив именно из содержимого каталога `backend`, включая скрытые файлы:
```sh
tar -C HAN_chat_specification/codebase/backend -czf han-chat-backend.tar.gz .
scp han-chat-backend.tar.gz deploy@<VM_IP>:/tmp/
```
На VM:
```sh
cd /opt/han-chat/backend
tar -xzf /tmp/han-chat-backend.tar.gz
# Обязательно при копировании с Windows:
find . -type f \( -name '*.sh' -o -name 'validate-env' \) -exec dos2unix {} +
chmod +x scripts/validate-env deployment/scripts/*.sh redis/scripts/*.sh nginx/scripts/*.sh
```
Проверьте наличие точки запуска:
```sh
test -f /opt/han-chat/backend/docker-compose.yml
test -f /opt/han-chat/backend/.env.example
```
## 5. Настройка DNS и сетевого доступа
Создайте DNS-запись:
```text
chat.example.ru A <ПУБЛИЧНЫЙ_IP_VM>
```
Дождитесь обновления DNS:
```sh
getent ahostsv4 chat.example.ru
```
В облачной группе безопасности VM разрешите входящие подключения:
- TCP 80 из интернета;
- TCP 443 из интернета;
- SSH только из доверенной сети или с административного IP.
Не открывайте наружу порты 6379, 4317, 4318, 8000, 8080 и 9000.
В группе безопасности PostgreSQL разрешите входящий трафик на порт PostgreSQL
только от приватного адреса или группы безопасности VM.
## 6. Подготовка управляемого PostgreSQL
Создайте одну базу данных:
```text
han_chat
```
В ней нужны схемы:
```text
han_app
bitrix_local
bitrix_sync
message_safety
keycloak
```
Для текущей MVP-реализации используются следующие пользователи:
```text
han_app
bitrix_local_app
bitrix_sync_user
message_safety_app
keycloak_user
```
Создать пользователей и схемы можно через панель провайдера либо от имени
администратора PostgreSQL. Пример SQL:
```sql
CREATE ROLE han_app LOGIN PASSWORD '<HAN_APP_PASSWORD>';
CREATE ROLE bitrix_local_app LOGIN PASSWORD '<BITRIX_LOCAL_PASSWORD>';
CREATE ROLE bitrix_sync_user LOGIN PASSWORD '<BITRIX_SYNC_PASSWORD>';
CREATE ROLE message_safety_app LOGIN PASSWORD '<SAFETY_PASSWORD>';
CREATE ROLE keycloak_user LOGIN PASSWORD '<KEYCLOAK_PASSWORD>';
CREATE SCHEMA IF NOT EXISTS han_app AUTHORIZATION han_app;
CREATE SCHEMA IF NOT EXISTS bitrix_local AUTHORIZATION bitrix_local_app;
CREATE SCHEMA IF NOT EXISTS bitrix_sync AUTHORIZATION bitrix_sync_user;
CREATE SCHEMA IF NOT EXISTS message_safety AUTHORIZATION message_safety_app;
CREATE SCHEMA IF NOT EXISTS keycloak AUTHORIZATION keycloak_user;
GRANT CONNECT ON DATABASE han_chat TO
han_app, bitrix_local_app, bitrix_sync_user, message_safety_app, keycloak_user;
```
Текущие migration jobs используют те же DSN, что и сервисы. Поэтому владельцы
схем должны иметь право создавать таблицы в своих схемах. Для более строгого
production-разделения migration/runtime ролей потребуется отдельная настройка
DSN и прав, которой в текущем `.env.example` нет.
Скачайте CA-сертификат PostgreSQL у провайдера и поместите его на VM:
```sh
mkdir -p /opt/han-chat/backend/secrets/pg
cp /путь/к/ca.pem /opt/han-chat/backend/secrets/pg/ca.pem
chmod 644 /opt/han-chat/backend/secrets/pg/ca.pem
```
CA-сертификат не является секретом. Права `644` нужны, чтобы его могли прочитать
контейнеры, работающие не от root.
Проверьте сетевую доступность:
```sh
nc -vz <PG_HOST> 6432
```
Замените `6432` на фактический порт провайдера.
## 7. Подготовка Selectel S3
Создайте три приватных бакета:
```text
han-chat-quarantine
han-chat-attachments
han-chat-documents
```
Создайте две пары ключей:
1. Ключ API с правом чтения и записи в бакеты.
2. Отдельный ключ Message Safety только с правом чтения карантина.
Для бакетов запретите публичный доступ. Для браузерной загрузки настройте CORS:
- Allowed origin: `https://chat.example.ru`;
- Methods: `PUT`, `GET`, `HEAD`;
- Headers: `Content-Type`, `x-amz-*`;
- Expose header: `ETag`.
Для карантина задайте lifecycle удаления объектов с запасом относительно
`MESSAGE_SAFETY_TASK_TTL_SEC`.
## 8. Создание файла окружения
`.env` содержит только несекретную конфигурацию. На VM:
```sh
cd /opt/han-chat/backend
umask 077
cp .env.example .env
chmod 600 .env
nano .env
```
Замените несекретные адреса `example.*`. Не добавляйте в `.env` пароли, токены,
ключи, credential-bearing DSN или пути `*_FILE`. Установите отдельно проверенный
launcher `deployment/secrets/han-secrets` из ops-пакета. Его интерфейс:
```sh
deployment/secrets/han-secrets run --config .env -- <command>
```
Launcher читает `SECRETS_SOURCE=file|selectel`, устанавливает
`HAN_SECRETS_ACTIVE=1`, выдаёт значения только дочернему процессу и не печатает
их. Рекомендуемый `HAN_RUNTIME_SECRET_MANIFEST` содержит только пары
`SECRET_KEY=/absolute/protected/path`; файлы имеют mode `0400`/`0600`.
### 8.1. Основные адреса
Для домена `chat.example.ru`:
```dotenv
APP_ENV=production-like
RELEASE_VERSION=2026-07-13-1
PUBLIC_HOST=chat.example.ru
PUBLIC_WEB_URL=https://chat.example.ru
PUBLIC_API_URL=https://chat.example.ru/api
PUBLIC_AUTH_URL=https://chat.example.ru/auth
KEYCLOAK_PUBLIC_URL=https://chat.example.ru/auth
KEYCLOAK_INTERNAL_URL=http://keycloak:8080/auth
KEYCLOAK_REALM=han-chat
KEYCLOAK_AUDIENCE=han-chat-api
```
### 8.2. PostgreSQL
В `.env` укажите только host, port, database и путь к публичному CA:
```dotenv
HAN_PG_HOST=<PG_HOST>
HAN_PG_PORT=6432
HAN_PG_DATABASE=han_chat
PG_CA_HOST_PATH=/opt/han-chat/backend/secrets/pg/ca.pem
KEYCLOAK_DB_SCHEMA=keycloak
```
Все service DSN, включая JDBC и backup DSN, формирует secret backend. Runtime
validator проверяет `verify-full`, `sslrootcert` и запрет `options/currentSchema`
без вывода строк подключения.
Порт `5433` используется с PgBouncer в режиме `session`. Не добавляйте
`options=-csearch_path...` или JDBC-параметр `currentSchema`: они передают
startup parameter `search_path`, который Selectel PgBouncer отклоняет. Для
Keycloak схема задаётся отдельно через `KEYCLOAK_DB_SCHEMA`.
Для каждой сервисной роли заранее задайте database-level `search_path`.
Если пароль содержит `@`, `:`, `/`, `?`, `#` или `%`, его необходимо
URL-кодировать внутри PostgreSQL URL.
### 8.3. Подготовка runtime-секретов
Генерируйте секреты вне shell history средствами secret manager. Не выполняйте
`export TOKEN=...` и не вставляйте значения в команды. Имена обязательных
runtime-переменных определены в `scripts/validate-env`; парные токены связываются
в secret backend.
```sh
openssl rand -hex 32
```
Для `BITRIX_TOKEN_ENCRYPTION_KEY` нужен URL-safe Base64 ключ ровно из 32 байт:
```sh
python3 -c 'import base64,secrets; print(base64.urlsafe_b64encode(secrets.token_bytes(32)).decode())'
```
Ни один из этих секретов не добавляется в `.env`. Пары проверяются runtime
validator без вывода значений.
```dotenv
BITRIX_LOCAL_APP_INTERNAL_TOKEN=<TOKEN_A>
BITRIX_INTERNAL_API_TOKEN=<TOKEN_A>
BITRIX_API_FORWARD_TOKEN=<TOKEN_B>
BITRIX_API_INBOX_TOKEN=<TOKEN_B>
```
Остальные токены должны быть разными:
```dotenv
MESSAGE_SAFETY_SERVICE_TOKEN=<UNIQUE_TOKEN>
BITRIX_SYNC_SERVICE_TOKEN=<UNIQUE_TOKEN>
KEYCLOAK_SETTINGS_BRIDGE_TOKEN=<UNIQUE_TOKEN>
CURSOR_HMAC_SECRET=<UNIQUE_TOKEN>
KEYCLOAK_OTP_HMAC_KEY=<UNIQUE_TOKEN_НЕ_КОРОЧЕ_32_БАЙТ>
BITRIX_TOKEN_ENCRYPTION_KEY=<URLSAFE_BASE64_KEY>
KEYCLOAK_ADMIN_PASSWORD=<UNIQUE_ADMIN_PASSWORD>
```
### 8.4. Redis
Создайте три разных пароля в secret backend; там же сформируйте Redis URL.
Следующий блок описывает логический контракт и не является содержимым `.env`:
```dotenv
REDIS_API_PASSWORD=<REDIS_API_PASSWORD>
REDIS_SAFETY_PASSWORD=<REDIS_SAFETY_PASSWORD>
REDIS_HEALTH_PASSWORD=<REDIS_HEALTH_PASSWORD>
REDIS_URL=redis://api_backend:<REDIS_API_PASSWORD>@redis:6379/0
REDIS_REALTIME_URL=redis://api_backend:<REDIS_API_PASSWORD>@redis:6379/1
MESSAGE_SAFETY_REDIS_URL=redis://message_safety:<REDIS_SAFETY_PASSWORD>@redis:6379/2
```
### 8.5. Mock OTP
В текущих deploy-артефактах реализован только mock OTP. Для запуска до controlled SMS rollout:
```dotenv
KEYCLOAK_OTP_MOCK_ENABLED=true
KEYCLOAK_OTP_MOCK_RISK_ACCEPTED=true
```
`KEYCLOAK_OTP_MOCK_CODE` хранится только в secret backend. Не используйте mock
как production-механизм доставки OTP.
Целевой real mode задаёт `modules/module-11-idgtl-sms.md`: Keycloak генерирует и локально проверяет OTP, `sms-service` надёжно записывает заказ/журнал, worker вызывает i-Digital Direct, callback обновляет только delivery journal. Нельзя просто установить `KEYCLOAK_OTP_MOCK_ENABLED=false`.
До переключения необходимы: schema/role `sms` и migrations/seed, active approved `auth_otp` (`code`, `ttl_min`), согласованный sender, Direct `TOKEN_1`, парные service tokens, отдельные callback credentials, exact nginx callback route, подтверждённый source IP Direct и статический egress IP worker. Сначала deploy при mock=true, затем provider smoke/callback/redaction evidence и только после этого cutover. Rollback возвращает mock без удаления SMS schema/journal.
### 8.6. S3
```dotenv
SELECTEL_S3_ENDPOINT_URL=https://s3.storage.selcloud.ru
SELECTEL_S3_BUCKET_QUARANTINE=han-chat-quarantine
SELECTEL_S3_BUCKET_ATTACHMENTS=han-chat-attachments
SELECTEL_S3_BUCKET_DOCUMENTS=han-chat-documents
```
Обе пары S3 credentials хранятся только в secret backend.
API backend принудительно использует virtual-hosted addressing:
`https://<bucket>.s3.storage.selcloud.ru/<object-key>`. Это обязательно для
браузерных presigned PUT и CORS в Selectel; path-style URL для этого сценария не
используйте. DNS и исходящий HTTPS с ВМ должны разрешать поддомены бакетов.
### 8.7. Bitrix24
До установки локального приложения загрузите credentials в secret backend.
В `.env` остаются только несекретные connector/public URL параметры:
```dotenv
BITRIX_CONNECTOR_ID=han_mobile_app
BITRIX_OPEN_LINE_ID=8
BITRIX_PUBLIC_BASE_URL=https://chat.example.ru/bitrix
```
`BITRIX_APPLICATION_TOKEN` сохраняется в secret backend после создания
приложения и никогда не помещается в `.env`.
### 8.8. TLS и наблюдаемость
До выпуска сертификата оставьте в `.env` целевые значения:
```dotenv
NGINX_TLS_ENABLED=true
NGINX_TLS_CERTIFICATE=/etc/letsencrypt/live/chat.example.ru/fullchain.pem
NGINX_TLS_CERTIFICATE_KEY=/etc/letsencrypt/live/chat.example.ru/privkey.pem
ACME_EMAIL=<ADMIN_EMAIL>
```
Если удаленный OTLP-бэкенд пока не выбран, укажите временные непубличные значения
и примите ограничение: Collector будет пытаться отправлять телеметрию и сохранять
ее в ограниченной очереди. Перед production-запуском задайте реальный endpoint.
## 9. Проверка окружения и конфигурации Compose
Выполните:
```sh
cd /opt/han-chat/backend
./scripts/validate-env .env
sudo systemctl restart han-secrets@production.service
sudo ./scripts/validate-env .env \
--runtime-manifest /run/han-chat/secrets/manifest
sudo deployment/secrets/han-compose config --quiet
sudo deployment/secrets/han-compose config --services
python3 -m unittest discover -s tests -v
```
Не переходите к следующему шагу, пока все команды не завершатся успешно.
Посмотрите итоговую конфигурацию портов:
```sh
docker compose --env-file .env config | grep -n 'published:'
```
Публиковаться должны только 80 и 443 у nginx.
## 10. Сборка образов
Соберите все локальные образы:
```sh
cd /opt/han-chat/backend
docker compose --env-file .env build --pull
```
Проверьте список:
```sh
docker compose --env-file .env images
```
Сборка Keycloak включает Java OTP SPI, а сборка `frontend-static` экспортирует
тестовый Expo Web frontend.
## 11. Миграции БД и начальные настройки
Перед миграциями создайте backup/PITR marker в панели провайдера PostgreSQL.
Затем:
```sh
cd /opt/han-chat/backend
PITR_MARKER_CONFIRMED=true deployment/scripts/migrate.sh
deployment/scripts/seed.sh
```
Скрипт применит миграции `han_app`, `bitrix_local` и baseline `bitrix_sync`, после
чего загрузит `deployment/app-settings.production-like.yaml`.
Повторный запуск seed должен быть безопасным:
```sh
deployment/scripts/seed.sh
```
## 12. Запуск внутренних сервисов
Сначала запустите Redis:
```sh
docker compose --env-file .env up -d redis
docker compose --env-file .env ps redis
```
Затем Keycloak и OpenTelemetry:
```sh
docker compose --env-file .env up -d keycloak otel-collector
docker compose --env-file .env ps keycloak otel-collector
```
Первый запуск Keycloak может занять несколько минут: он создаст свои таблицы и
импортирует realm `han-chat`.
После готовности Keycloak:
```sh
docker compose --env-file .env up -d message-safety
docker compose --env-file .env up -d api-backend
docker compose --env-file .env up -d bitrix-local-app bitrix-sync
docker compose --env-file .env up -d \
delivery-worker safety-recovery-worker cleanup-worker
docker compose --env-file .env ps
```
Если сервис не становится healthy:
```sh
docker compose --env-file .env logs --tail=200 <SERVICE_NAME>
docker inspect "$(docker compose --env-file .env ps -q <SERVICE_NAME>)"
```
### 12.1. Повторная раскатка upstream при уже работающем nginx
Nginx разрешает Docker DNS имена upstream при загрузке конфигурации. После
`up --build`, `pull`, rollback или `--force-recreate` контейнер может получить
новый IP, а работающий nginx продолжит использовать старый и вернёт `502
Connection refused`.
После пересоздания `api-backend`, `keycloak`, `sms-service` или
`bitrix-local-app` обязательно выполните:
```sh
docker compose --env-file .env up -d --wait \
api-backend keycloak sms-service bitrix-local-app
docker compose --env-file .env exec -T nginx \
nginx -t -c /tmp/nginx.conf
docker compose --env-file .env kill -s HUP nginx
curl -fsS "https://${PUBLIC_HOST}/api/v1/public/app-config" | jq
curl -fsS \
"https://${PUBLIC_HOST}/auth/realms/han-chat/.well-known/openid-configuration" |
jq
```
Не используйте bare-команды `nginx -t` и `nginx -s reload`: рабочая
конфигурация находится в `/tmp/nginx.conf`, PID — в `/tmp/nginx.pid`, а
контейнер использует read-only filesystem.
## 13. Первоначальный выпуск TLS-сертификата
Для ACME требуется работающий nginx по HTTP. В `.env` оставьте
`NGINX_TLS_ENABLED=true`, но первый nginx запустите с временным переопределением:
```sh
NGINX_TLS_ENABLED=false \
docker compose --env-file .env up -d frontend-static nginx
```
Проверьте HTTP:
```sh
curl -I http://chat.example.ru/
```
Сначала рекомендуется проверить Certbot через staging:
```sh
docker compose --env-file .env --profile certbot run --rm certbot certonly \
--staging \
--webroot -w /var/www/certbot \
-d chat.example.ru \
--cert-name chat.example.ru-staging \
--email <ADMIN_EMAIL> \
--agree-tos --no-eff-email --non-interactive
```
После успешного staging-теста выпустите рабочий сертификат с основным cert-name
без `--staging`:
```sh
docker compose --env-file .env --profile certbot run --rm certbot certonly \
--webroot -w /var/www/certbot \
-d chat.example.ru \
--cert-name chat.example.ru \
--email <ADMIN_EMAIL> \
--agree-tos --no-eff-email --non-interactive
```
Пересоздайте nginx уже с TLS:
```sh
docker compose --env-file .env up -d --force-recreate nginx
docker compose --env-file .env exec -T nginx nginx -t -c /tmp/nginx.conf
curl -I https://chat.example.ru/
```
Повторно запустите VM setup, чтобы он обнаружил проект и установил systemd-таймер
продления сертификата:
```sh
sudo /opt/han-chat/backend/deployment/scripts/setup-vm.sh
systemctl status han-chat-ssl-renew.timer
```
## 14. Запуск всего контура
Теперь можно привести весь проект к состоянию, описанному Compose:
```sh
cd /opt/han-chat/backend
docker compose --env-file .env up -d
docker compose --env-file .env ps
```
Проверьте, что контейнеры не перезапускаются:
```sh
docker compose --env-file .env ps
docker compose --env-file .env logs --since=10m
```
## 15. Публичная проверка
Запустите smoke-тест:
```sh
cd /opt/han-chat/backend
deployment/scripts/smoke.sh
```
Также вручную проверьте:
```sh
curl -fsS https://chat.example.ru/api/v1/public/app-config | jq
curl -fsS https://chat.example.ru/api/v1/public/content | jq
curl -fsS \
https://chat.example.ru/auth/realms/han-chat/.well-known/openid-configuration | jq
```
Внутренний API не должен быть опубликован:
```sh
curl -i https://chat.example.ru/internal/safety/v2/messages/check
```
Ожидаемый статус — `404`.
Откройте в браузере:
```text
https://chat.example.ru/
```
Для тестовой авторизации получите mock code утверждённым защищённым способом,
не читая его из `.env` и не помещая в shell history.
## 16. Подключение Bitrix24
В настройках локального приложения Bitrix24 задайте HTTPS-адреса:
```text
Установка: https://chat.example.ru/bitrix/install
Обработчик: https://chat.example.ru/bitrix/handler
Placement: https://chat.example.ru/bitrix/placement
```
После установки:
1. Получите и сохраните application token.
2. Сохраните его как `BITRIX_APPLICATION_TOKEN` в secret backend.
3. Пересоздайте сервис:
```sh
docker compose --env-file .env up -d --force-recreate bitrix-local-app
docker compose --env-file .env logs --tail=200 bitrix-local-app
```
Проверьте коннектор `han_mobile_app` и Открытую линию 8.
## 17. Включение SSH hardening
Только после успешного входа пользователем `deploy` по ключу в отдельной сессии:
```sh
sudo HARDEN_SSH=true \
/opt/han-chat/backend/deployment/scripts/setup-vm.sh
```
Не закрывайте текущую SSH-сессию, пока не проверили новый вход.
## 18. Обычный перезапуск проекта
Для штатного запуска после перезагрузки VM:
```sh
cd /opt/han-chat/backend
docker compose --env-file .env up -d
docker compose --env-file .env ps
```
Для перезапуска одного сервиса:
```sh
docker compose --env-file .env restart api-backend
```
После изменения `.env` используйте пересоздание, а не `restart`:
```sh
docker compose --env-file .env up -d --force-recreate <SERVICE_NAME>
```
## 19. Обновление версии проекта
Перед обновлением:
1. Создайте backup/PITR marker PostgreSQL.
2. Сохраните текущие image digests.
3. Получите новый код.
4. Проверьте несекретный `.env` и runtime secret set.
5. Пересоберите образы.
6. Примените миграции и seed.
7. Пересоздайте сервисы.
Команды:
```sh
cd /opt/han-chat/backend
./scripts/validate-env .env
docker compose --env-file .env build --pull
PITR_MARKER_CONFIRMED=true deployment/scripts/migrate.sh
deployment/scripts/seed.sh
docker compose --env-file .env up -d
deployment/scripts/smoke.sh
```
## 20. Диагностика
Состояние сервисов:
```sh
docker compose --env-file .env ps
```
Все логи:
```sh
docker compose --env-file .env logs --tail=300
```
Логи конкретного сервиса:
```sh
docker compose --env-file .env logs -f api-backend
```
Проверка firewall:
```sh
sudo ufw status verbose
sudo iptables -L HAN-CHAT-DOCKER -n -v
```
Проверка сертификата:
```sh
openssl s_client -connect chat.example.ru:443 -servername chat.example.ru \
</dev/null 2>/dev/null | openssl x509 -noout -subject -issuer -dates
```
Проверка свободного места:
```sh
df -h
docker system df
```
Не выполняйте `docker compose down -v`: эта команда удалит именованные volumes.
Не выполняйте Alembic downgrade. Для отката используйте
`deployment/scripts/rollback.sh` и инструкции из `RUNBOOK.ru.md`.
## 21. Когда развертывание можно считать завершенным
Проект запущен корректно, если:
- `docker compose ps` показывает healthy для критических сервисов;
- `deployment/scripts/smoke.sh` завершается успешно;
- открывается тестовый frontend;
- проходит авторизация через mock OTP;
- отправляются текстовые и файловые сообщения;
- внутренние URL возвращают 404 снаружи;
- TLS-сертификат действителен;
- логи не содержат токены, PII и тексты сообщений;
- настроены резервное копирование PostgreSQL и продление TLS.
Для формальной production-like приемки после этого пройдите контрольные этапы
из `deployment/RUNBOOK.ru.md`.
VM1_app/codebase/backend/deployment/RUNBOOK.md
+330
View File
@@ -0,0 +1,330 @@
# HAN Chat production-like deployment runbook
This is the executable checklist for the single-VM contour. PostgreSQL and S3
are managed external services. Never use `docker compose down -v`, an Alembic
downgrade, or a mutable image tag during deployment.
## VM2 Processing is a separate host
Do not run this backend/VM1 setup script on VM2. VM2 has its own bootstrap:
`codebase/services/deployment/scripts/setup-vm.sh`, and its authoritative
operator checklist is `codebase/services/deployment/RUNBOOK.ru.md`.
The VM2 ownership boundary is intentionally different from the legacy VM1
script: `deploy` is **not** a member of the `docker` group. Root owns
`/opt/han-chat/services`, Compose, units, helpers, `.env`, allow-lists and
secret mappings. Deploy may write only to `/var/lib/han-deploy/incoming` and
may invoke exact systemd/safety-mode commands installed in sudoers.
The separate `admin` account is break-glass only: it has its own Ed25519 key
and a separate local sudo password. Root, deploy and admin keys must differ.
Initial VM2 bootstrap commands:
```sh
# Local operator workstation: upload only the reviewed setup script.
scp codebase/services/deployment/scripts/setup-vm.sh \
root@<VM2_PUBLIC_IP>:/root/setup-vm2.sh
# VM2 root: install host packages/roles/firewalls; this does not start Compose.
chmod 0700 /root/setup-vm2.sh
DEPLOY_AUTHORIZED_KEY_FILE=/root/bootstrap/deploy.pub \
ADMIN_AUTHORIZED_KEY_FILE=/root/bootstrap/admin.pub \
OPS_CIDRS='<OPS_PUBLIC_IP>/32' \
VM1_PRIVATE_CIDRS='<VM1_PRIVATE_IP>/32' \
/root/setup-vm2.sh
```
Generate and upload the two public keys before this command; never copy the
root key into either account. Set the admin sudo password with `passwd admin`.
Keep the root session open and verify both key-based logins plus `sudo -v` as
admin in separate sessions. Only then rerun as VM2 root with
`HARDEN_SSH=true SKIP_APT_UPGRADE=true` to disable direct root SSH.
Release transfer is performed as deploy, while activation and installation
remain root operations:
```sh
# deploy: receive and inspect only.
cd /var/lib/han-deploy/incoming
sha256sum vm2-services-<RELEASE>.tar.gz
tar -tzf vm2-services-<RELEASE>.tar.gz
# root: verify the operator-provided digest, activate root-owned files,
# then rerun setup-vm.sh so it installs fixed helpers and systemd units.
printf '%s %s\n' '<EXPECTED_SHA256>' \
/var/lib/han-deploy/incoming/vm2-services-<RELEASE>.tar.gz | sha256sum --check -
ARCHIVE=/var/lib/han-deploy/incoming/vm2-services-<RELEASE>.tar.gz
if tar -tzf "$ARCHIVE" | grep -Eq '(^/|(^|/)\.\.(/|$)|^services/\.env$)'; then exit 1; fi
if tar -tzf "$ARCHIVE" | grep -Ev '^services(/|$)' | grep -q .; then exit 1; fi
if tar -tvzf "$ARCHIVE" | awk '$1 ~ /^[lh]/ {found=1} END {exit !found}'; then exit 1; fi
STAGING="$(mktemp -d /opt/han-chat/.vm2-release.XXXXXX)"
tar -xzf "$ARCHIVE" \
-C "$STAGING" --no-same-owner --no-same-permissions
test -f "$STAGING/services/docker-compose.yml"
rsync -a --delete --exclude=.env --chown=root:root --chmod=D755,F644 \
"$STAGING/services/" /opt/han-chat/services/
rm -rf -- "$STAGING"
OPS_CIDRS='<OPS_PUBLIC_IP>/32' \
VM1_PRIVATE_CIDRS='<VM1_PRIVATE_IP>/32' \
DEPLOY_AUTHORIZED_KEY_FILE=/root/bootstrap/deploy.pub \
ADMIN_AUTHORIZED_KEY_FILE=/root/bootstrap/admin.pub \
HARDEN_SSH=true SKIP_APT_UPGRADE=true \
/root/setup-vm2.sh
```
After root configures `.env`, Selectel encrypted credentials, loader mapping,
TLS and CIDR allow-lists, root synchronizes secrets, runs preflight/migrations
and performs the first start. Subsequent routine operations available to
deploy are limited to:
```sh
sudo systemctl restart han-secrets-vm2.service
sudo systemctl restart han-processing.service
sudo systemctl --no-pager status han-processing.service
sudo journalctl --no-pager -u han-processing.service
```
Exact archive activation, file installation, credential creation, migration
and first-start commands are documented in the VM2 Russian runbook referenced
above. They must not be replaced with direct Docker access for deploy.
## Gate 0 — decisions and ownership
- [ ] Release SHA/digests, maintenance window, on-call and rollback owner recorded.
- [ ] RPO/RTO accepted; initial targets are PG RPO <=15 minutes and RTO <=4 hours.
- [ ] Remote OTLP backend selected, or debug-only acceptance limitation accepted.
- [ ] Mock OTP, Safety stub and bitrix-sync stub risks explicitly accepted.
## Gate 1 — VPC, DNS and security groups
- [ ] Managed PostgreSQL has only a private endpoint and accepts traffic from VM SG.
- [ ] Internet can reach only VM TCP 80/443; SSH is restricted to VPN/ops CIDR.
- [ ] Ports 6379, 4317/4318, 8000, 8080 and 9000 are denied externally.
- [ ] DNS `A` for `PUBLIC_HOST` points at the VM and outbound HTTPS is available.
## Gate 2 — VM hardening
On a fresh Ubuntu 24.04 VM, run:
```sh
sudo deployment/scripts/setup-vm.sh
```
The script disables password SSH and X11 forwarding by default, then locks the
local `root` and `deploy` passwords after checking authorized keys. Before
setting `HARDEN_SSH=true`, which also disables root login and TCP forwarding,
verify key-based deploy access in a separate SSH session.
- [ ] Ubuntu 24.04, NTP, unattended security updates and disk alerts are active.
- [ ] Key-only deploy account works in a second session; root/password SSH is off.
- [ ] UFW/cloud SG and `DOCKER-USER` policy survive reboot.
- [ ] Docker Engine and Compose support `include` and long-form `env_file`.
## Gate 3 — managed PostgreSQL
- [ ] Daily backup, PITR, deletion protection, encryption and alerts are enabled.
- [ ] Provider CA is installed at `PG_CA_HOST_PATH`; all DSNs use `verify-full`.
- [ ] Schemas `han_app`, `bitrix_local`, `bitrix_sync`, `message_safety`, `keycloak`
have separate migration/runtime roles with tested negative grants.
- [ ] Migration tested against an empty DB and a clone of the previous release.
## Gate 4 — Selectel S3
- [ ] Quarantine, attachments and documents buckets are private and encrypted.
- [ ] API credentials are prefix-scoped; Safety credentials are quarantine read-only.
- [ ] Browser CORS permits exact HTTPS origin and PUT headers only.
- [ ] Quarantine lifecycle exceeds Safety poll/recovery; data retention is approved.
## Gate 5 — immutable release
- [ ] Checkout is detached at the approved SHA and working tree is clean.
- [ ] Service images are immutable and scanned; no unresolved critical/high issue.
- [ ] Root `docker-compose.yml` is the only deployment entry point.
## Gate 6 — environment and secrets
```sh
cp .env.example .env
# Replace non-secret configuration placeholders only.
./scripts/validate-env .env
sudo systemctl restart han-secrets@production.service
sudo ./scripts/validate-env .env \
--runtime-manifest /run/han-chat/secrets/manifest
sudo deployment/secrets/han-compose config --quiet
```
- [ ] `SECRETS_SOURCE=file|selectel`; `.env` contains no secret keys or credential-bearing DSNs.
- [ ] `deployment/secrets/han-secrets` sets `HAN_SECRETS_ACTIVE=1`, does not log values,
and optionally exposes a paths-only `HAN_RUNTIME_SECRET_MANIFEST`.
- [ ] Runtime token pairs match, PG verifies TLS, public URLs are HTTPS.
- [ ] Mock OTP risk is accepted and runtime secrets are unique >=128-bit values.
- [ ] `NOTIFICATIONS_TOKEN_PRODUCER_TEST` is unique and supplied only through secret/env; the `producer_test` source seed stores only its hash.
- [ ] `FRONTEND_DEV_PROXY_ENABLED=false` and Safety/nginx timeout budgets match.
## Gate 7 — images and static frontend
```sh
docker compose --env-file .env pull
docker compose --env-file .env build --pull frontend-static nginx redis
docker compose --env-file .env run --rm frontend-static
```
- [ ] Frontend export was tested/scanned and copied by `frontend-static` into its named volume.
- [ ] Build artifacts contain no secrets or unintended source maps.
- [ ] At least 30% VM disk remains free.
## Gate 8 — topology
```sh
docker compose --env-file .env config --services
python3 -m unittest discover -s tests -v
```
- [ ] Exactly nginx publishes `80:80` and `443:443`; no PostgreSQL service exists.
- [ ] Redis AOF/RDB/ACL and OTEL persistent queue volumes are present.
- [ ] `backend` and `observability` are internal networks.
## Gate 9 — ACME/TLS bootstrap
Set `NGINX_TLS_ENABLED=false` only for this bootstrap command:
```sh
NGINX_TLS_ENABLED=false docker compose --env-file .env up -d nginx
docker compose --profile certbot run --rm certbot certonly \
--webroot -w /var/www/certbot -d "$PUBLIC_HOST" \
--cert-name "$PUBLIC_HOST" --email "$ACME_EMAIL" \
--agree-tos --no-eff-email --non-interactive
docker compose --env-file .env up -d --force-recreate nginx
docker compose exec -T nginx nginx -t -c /tmp/nginx.conf
```
First rehearse with Certbot `--staging`. Install a twice-daily systemd timer for
`deployment/scripts/ssl-renew.sh`; test `certbot renew --dry-run`. Enable HSTS
only after chain, hostname, redirect and TLS 1.2/1.3 checks pass.
## Gate 10 — migrations and seed
Create a provider PITR marker, then:
```sh
PITR_MARKER_CONFIRMED=true deployment/scripts/migrate.sh
deployment/scripts/seed.sh
```
- [ ] Expected Alembic revisions are active and runtime users did not perform DDL.
- [ ] Seed succeeds twice and mandatory settings contain no secret.
- [ ] Schema remains backward-compatible with the previous images.
## Gate 11 — Keycloak
```sh
docker compose up -d keycloak
docker compose ps keycloak
```
- [ ] Discovery/JWKS issuer is the exact public `/auth` HTTPS URL.
- [ ] Frontend client is public PKCE S256; implicit/password/social flows are off.
- [ ] Wrong/replayed OTP and limits fail safely; settings bridge is fail-closed.
- [ ] Bootstrap admin was removed/rotated and named admin MFA is enabled.
## Gate 12 — ordered startup and readiness
```sh
docker compose up -d redis
docker compose up -d keycloak otel-collector
docker compose up -d message-safety
docker compose up -d api-backend
docker compose up -d delivery-worker safety-recovery-worker cleanup-worker \
notification-expire-worker notification-draft-cleanup-worker
docker compose up -d bitrix-local-app bitrix-sync
docker compose up -d nginx
docker compose up -d --wait api-backend keycloak sms-service bitrix-local-app
docker compose exec -T nginx nginx -t -c /tmp/nginx.conf
docker compose kill -s HUP nginx
docker compose ps
```
Nginx resolves Docker upstream names when its configuration is loaded. After
recreating `api-backend`, `keycloak`, `sms-service`, or `bitrix-local-app`,
wait for readiness, validate the active `/tmp/nginx.conf`, and signal the
master process with `HUP` as shown above. Do not use bare `nginx -t` or
`nginx -s reload`: they target the default config/PID under read-only
`/var/run`, not the running Nginx instance.
- [ ] No restart loop/OOM; critical readiness is green.
- [ ] `notification-expire-worker` runs daily closure with an advisory lock; `notification-draft-cleanup-worker` removes expired drafts/S3 objects. Both entrypoints exist in the installed image.
- [ ] Only documented Bitrix not-installed/sync-stub degradation remains.
- [ ] External `/internal/*` is 404 and OTEL accepts telemetry.
## Gate 13 — Bitrix24
- [ ] Install, handler and placement URLs use the exact public HTTPS paths.
- [ ] Connector `han_mobile_app` is active on Open Line 8; events are bound once.
- [ ] OAuth is encrypted; callback/application/service tokens never enter logs.
- [ ] Outbound and operator reply paths are idempotent; internal status is private.
## Gate 14 — smoke and E2E
```sh
deployment/scripts/smoke.sh
```
- [ ] Guest, OTP/PKCE/bootstrap/session, refresh and logout paths pass.
- [ ] Safety allow/deny/pending/timeout and one concurrent slow poll pass.
- [ ] File quarantine/promote/delete, owner-only download and audit pass.
- [ ] WS reconnect plus REST reconciliation, ownership 404, idempotency and 429 pass.
- [ ] Closed-network `producer_test` Create/Cancel smoke passes; identical Create returns `200`, changed payload returns `409`, and the external internal route returns `404`.
- [ ] Expire advisory locking and first download of any linked document are verified; hiding is one-time and an existing `date_expired` is preserved.
- [ ] Logs contain no PII, message body, token or presigned query.
## Gate 15 — observability
- [ ] Known request ID links nginx, API and downstream trace; UX ID is not a label.
- [ ] Three signals reach the selected backend; SLO queries and alerts are tested.
- [ ] Remote outage fills/drains the bounded persistent queue without business outage.
- [ ] Secret/PII canary is absent. Collector restart/drop/refused metrics are checked.
For local acceptance only, start the redacted debug collector with:
`docker compose --profile observability-local up -d otel-collector-local`.
## Gate 16 — open traffic
- [ ] Gates 0–15 are signed; fresh backup/PITR evidence and previous images exist.
- [ ] HSTS is enabled, release digests/schema/realm versions are recorded.
- [ ] No active page; on-call and product owner accept stub limitations.
- [ ] Observe 5xx/auth/delivery/DB/Redis/OOM/OTEL queue/Bitrix for 60 minutes.
## Backup and restore
Provider backup/PITR is authoritative. A supplemental verified logical dump:
```sh
deployment/scripts/backup.sh /opt/han-chat/backups
```
Quarterly, restore PG and S3 into an isolated VPC, deploy the same image digests,
do not route production DNS/Bitrix callbacks, run smoke, and record measured RPO/RTO.
Redis may be restored empty; its AOF/RDB is not a business backup.
## Rollback
Only roll back to images compatible with the current schema:
```sh
SCHEMA_BACKWARD_COMPATIBLE_CONFIRMED=true \
deployment/scripts/rollback.sh <PREVIOUS_IMMUTABLE_RELEASE>
deployment/scripts/smoke.sh
```
Rollback reuses the current runtime secret set and non-secret config. Do not
create or restore an environment snapshot.
## Real SMS rollout addendum
This runbook remains mock-only until module-11 artifacts exist. An SMS release requires schema/role `sms`, versioned migrations and an active approved `auth_otp` seed, `sms-service`/worker, the exact callback route, paired service tokens, Direct `TOKEN_1`, approved sender/template, separate callback credentials, a reconfirmed callback source IP, and a static worker egress IP.
Order: App DB OTP seed → SMS schema/migrations/seed → mock Direct tests → production SMS deployment while Keycloak remains in mock mode → Keycloak expand migration/SPI → controlled provider smoke plus callback/redaction evidence → real mode. Roll back by restoring mock mode without deleting the journal/schema; stop new real orders and drain or record in-flight/`uncertain` rows. Downgrade only with proven schema compatibility.
Never run Alembic downgrade. After a backward-incompatible migration choose a
forward fix or coordinated PITR/S3/Bitrix reconciliation under maintenance.
Always verify outbox/inbox/recovery so an ambiguous message is not sent twice.
VM1_app/codebase/backend/deployment/RUNBOOK.ru.md
+271
View File
@@ -0,0 +1,271 @@
# Инструкция по развертыванию HAN Chat в production-like окружении
Это исполняемый чек-лист для контура на одной виртуальной машине. PostgreSQL и S3
используются как внешние управляемые сервисы. Во время развертывания запрещено
использовать `docker compose down -v`, откат миграций Alembic и изменяемые теги образов.
Подробная пошаговая инструкция для первого запуска находится в
`deployment/DEPLOYMENT_GUIDE.ru.md`.
## Этап 0 — решения и зоны ответственности
- [ ] Зафиксированы SHA/дайджесты релиза, окно обслуживания, дежурный и ответственный за откат.
- [ ] Согласованы RPO/RTO; начальные цели: RPO PostgreSQL не более 15 минут и RTO не более 4 часов.
- [ ] Выбран удаленный OTLP-бэкенд либо принято ограничение на использование только отладочного контура.
- [ ] Явно приняты риски mock OTP, заглушки Safety и заглушки bitrix-sync.
## Этап 1 — VPC, DNS и группы безопасности
- [ ] Управляемый PostgreSQL имеет только приватную точку доступа и принимает трафик от группы безопасности VM.
- [ ] Из интернета доступны только TCP-порты VM 80/443; SSH ограничен VPN или CIDR администраторов.
- [ ] Порты 6379, 4317/4318, 8000, 8080 и 9000 закрыты для внешнего доступа.
- [ ] DNS-запись `A` для `PUBLIC_HOST` указывает на VM; исходящий HTTPS доступен.
## Этап 2 — защита виртуальной машины
На новой Ubuntu 24.04 можно выполнить подготовительный скрипт:
```sh
sudo deployment/scripts/setup-vm.sh
```
Скрипт по умолчанию отключает парольный SSH-вход и X11 forwarding, а после
проверки ключей блокирует локальные пароли `root` и `deploy`. Перед включением
`HARDEN_SSH=true`, которое дополнительно запрещает root-вход и TCP forwarding,
обязательно проверьте вход пользователем `deploy` по ключу в отдельной сессии.
- [ ] Установлена Ubuntu 24.04; работают NTP, автоматические обновления безопасности и оповещения о заполнении диска.
- [ ] Вход учетной записью развертывания по ключу проверен во второй сессии; вход root и SSH по паролю отключены.
- [ ] Правила UFW/облачной группы безопасности и политика `DOCKER-USER` сохраняются после перезагрузки.
- [ ] Docker Engine и Compose поддерживают `include` и полную форму `env_file`.
## Этап 3 — управляемый PostgreSQL
- [ ] Включены ежедневные резервные копии, PITR, защита от удаления, шифрование и оповещения.
- [ ] CA-сертификат провайдера установлен по пути `PG_CA_HOST_PATH`; все DSN используют `verify-full`.
- [ ] Для схем `han_app`, `bitrix_local`, `bitrix_sync`, `message_safety`, `keycloak`
созданы отдельные роли миграций и выполнения; запрет лишних прав проверен тестами.
- [ ] Миграции проверены на пустой БД и на клоне БД предыдущего релиза.
## Этап 4 — Selectel S3
- [ ] Бакеты карантина, вложений и документов закрыты от публичного доступа и зашифрованы.
- [ ] Права API ограничены префиксами; учетные данные Safety имеют доступ к карантину только на чтение.
- [ ] CORS бакетов разрешает только точный HTTPS-origin браузера и необходимые заголовки PUT.
- [ ] Срок хранения карантина превышает время Safety polling/recovery; политика хранения данных согласована.
## Этап 5 — неизменяемый релиз
- [ ] Репозиторий переключен на утвержденный SHA в detached-режиме; рабочее дерево чистое.
- [ ] Образы сервисов неизменяемы и просканированы; нерешенных критических и высоких уязвимостей нет.
- [ ] Корневой `docker-compose.yml` является единственной точкой запуска.
## Этап 6 — окружение и секреты
```sh
cp .env.example .env
# Замените только несекретные placeholders.
./scripts/validate-env .env
sudo systemctl restart han-secrets@production.service
sudo ./scripts/validate-env .env \
--runtime-manifest /run/han-chat/secrets/manifest
sudo deployment/secrets/han-compose config --quiet
```
- [ ] `SECRETS_SOURCE=file|selectel`; `.env` не содержит secret keys и DSN с credentials.
- [ ] `deployment/secrets/han-secrets` устанавливает `HAN_SECRETS_ACTIVE=1`,
не пишет значения в лог и при возможности передаёт paths-only manifest
через `HAN_RUNTIME_SECRET_MANIFEST`.
- [ ] Runtime-пары токенов совпадают, PostgreSQL проверяет TLS.
- [ ] Риск mock OTP принят; runtime-секреты уникальны и содержат не менее 128 бит энтропии.
- [ ] `NOTIFICATIONS_TOKEN_PRODUCER_TEST` сгенерирован отдельно, передан только через secret/env; seed `notification_sources.code='producer_test'` содержит только его hash.
- [ ] Установлено `FRONTEND_DEV_PROXY_ENABLED=false`; таймауты Safety и nginx согласованы.
## Этап 7 — образы и статический frontend
```sh
docker compose --env-file .env pull
docker compose --env-file .env build --pull frontend-static nginx redis
docker compose --env-file .env run --rm frontend-static
```
- [ ] Экспорт frontend проверен и просканирован, затем скопирован сервисом `frontend-static` в именованный volume.
- [ ] Артефакты сборки не содержат секретов и непредусмотренных source map.
- [ ] На диске VM остается не менее 30% свободного места.
## Этап 8 — топология
```sh
docker compose --env-file .env config --services
python3 -m unittest discover -s tests -v
```
- [ ] Только nginx публикует `80:80` и `443:443`; сервиса PostgreSQL в Compose нет.
- [ ] Присутствуют volumes Redis AOF/RDB/ACL и постоянной очереди OTEL.
- [ ] Сети `backend` и `observability` являются внутренними.
## Этап 9 — первоначальная настройка ACME/TLS
Установите `NGINX_TLS_ENABLED=false` только для команды первоначального запуска:
```sh
NGINX_TLS_ENABLED=false docker compose --env-file .env up -d nginx
docker compose --profile certbot run --rm certbot certonly \
--webroot -w /var/www/certbot -d "$PUBLIC_HOST" \
--cert-name "$PUBLIC_HOST" --email "$ACME_EMAIL" \
--agree-tos --no-eff-email --non-interactive
docker compose --env-file .env up -d --force-recreate nginx
docker compose exec -T nginx nginx -t -c /tmp/nginx.conf
```
Сначала выполните проверку с параметром Certbot `--staging`. Установите systemd-таймер,
запускающий `deployment/scripts/ssl-renew.sh` дважды в сутки, и проверьте
`certbot renew --dry-run`. Включайте HSTS только после проверки цепочки сертификатов,
имени хоста, перенаправления и поддержки TLS 1.2/1.3.
## Этап 10 — миграции и начальные данные
Создайте у провайдера точку восстановления PITR, затем выполните:
```sh
PITR_MARKER_CONFIRMED=true deployment/scripts/migrate.sh
deployment/scripts/seed.sh
```
- [ ] Активны ожидаемые ревизии Alembic; runtime-пользователи не выполняли DDL.
- [ ] Повторный seed завершается успешно; обязательные настройки не содержат секретов.
- [ ] Схема остается обратно совместимой с образами предыдущего релиза.
## Этап 11 — Keycloak
```sh
docker compose up -d keycloak
docker compose ps keycloak
```
- [ ] Issuer discovery/JWKS точно совпадает с публичным HTTPS URL `/auth`.
- [ ] Frontend-клиент является публичным PKCE S256; implicit, password и social flows отключены.
- [ ] Неверный или повторно использованный OTP и превышение лимитов безопасно отклоняются; settings bridge работает fail-closed.
- [ ] При `KEYCLOAK_YANDEX_CAPTCHA_ENABLED=true` initial send и resend требуют свежий SmartCaptcha token; техническая недоступность Yandex подтверждена как fail-open в логах.
- [ ] CSP login-страницы содержит `smartcaptcha.cloud.yandex.ru`/`yastatic.net`, а `/auth/realms/master/protocol/openid-connect/3p-cookies/step2.html` и Admin Console работают без CAPTCHA CSP.
- [ ] Временный администратор удален либо его пароль изменен; для именного администратора включена MFA.
Если предыдущая попытка сохранила custom CSP в realm, сбросьте только это поле через `kcadm`; `.env` как shell-файл не загружать:
```sh
docker compose exec -T keycloak sh -lc '
set -eu
cfg=/tmp/han-kcadm.config
/opt/keycloak/bin/kcadm.sh config credentials --config "$cfg" \
--server http://127.0.0.1:8080/auth --realm master \
--user "$KC_BOOTSTRAP_ADMIN_USERNAME" \
--password "$KC_BOOTSTRAP_ADMIN_PASSWORD"
/opt/keycloak/bin/kcadm.sh update realms/han-chat --config "$cfg" \
-s "browserSecurityHeaders.contentSecurityPolicy="
rm -f "$cfg"
'
```
## Этап 12 — последовательный запуск и готовность
```sh
docker compose up -d redis
docker compose up -d keycloak otel-collector
docker compose up -d message-safety
docker compose up -d api-backend
docker compose up -d delivery-worker safety-recovery-worker cleanup-worker \
notification-expire-worker notification-draft-cleanup-worker
docker compose up -d bitrix-local-app bitrix-sync
docker compose up -d nginx
docker compose up -d --wait api-backend keycloak sms-service bitrix-local-app
docker compose exec -T nginx nginx -t -c /tmp/nginx.conf
docker compose kill -s HUP nginx
docker compose ps
```
Nginx разрешает Docker DNS имена upstream при загрузке конфигурации. После
любого пересоздания `api-backend`, `keycloak`, `sms-service` или
`bitrix-local-app` дождитесь их readiness, проверьте именно рабочий
`/tmp/nginx.conf` и отправьте master-процессу `HUP`, как показано выше.
Обычные `nginx -t` и `nginx -s reload` использовать нельзя: они обращаются к
дефолтному config/PID в read-only `/var/run` и не перезагружают рабочий Nginx.
- [ ] Нет циклических перезапусков и OOM; критические readiness-проверки успешны.
- [ ] `notification-expire-worker` выполняет ежедневное закрытие с advisory lock; `notification-draft-cleanup-worker` очищает просроченные drafts/S3. Оба entrypoint присутствуют в установленном образе.
- [ ] Сохраняется только документированная деградация: Bitrix не установлен и bitrix-sync работает как заглушка.
- [ ] Внешний запрос `/internal/*` возвращает 404; OTEL принимает телеметрию.
## Этап 13 — Bitrix24
- [ ] URL установки, обработчика и placement используют точные публичные HTTPS-пути.
- [ ] Коннектор `han_mobile_app` активен в Открытой линии 8; события привязаны однократно.
- [ ] OAuth зашифрован; callback-, application- и service-токены не попадают в логи.
- [ ] Исходящие сообщения и ответы оператора идемпотентны; внутренний статус не опубликован наружу.
## Этап 14 — smoke- и E2E-тесты
```sh
deployment/scripts/smoke.sh
```
- [ ] Успешны сценарии гостя, OTP/PKCE/bootstrap/session, обновления токена и выхода.
- [ ] Проверены Safety allow/deny/pending/timeout и один параллельный медленный poll.
- [ ] Проверены карантин, перенос и удаление файлов, скачивание только владельцем и аудит.
- [ ] Проверены переподключение WS с REST-сверкой, 404 при обращении к чужому ресурсу, идемпотентность и 429.
- [ ] От имени `producer_test` выполнены Create и Cancel через закрытый `/internal/notifications/v1/*`; тот же Create вернул `200`, изменённый payload — `409`, внешний запрос — `404`.
- [ ] Проверены expire job с advisory lock и первое скачивание любого связанного документа: уведомление скрывается один раз, а исходный `date_expired` не перезаписывается.
- [ ] Логи не содержат PII, текстов сообщений, токенов и query-параметров presigned URL.
## Этап 15 — наблюдаемость
- [ ] Известный request ID связывает трассировку nginx, API и downstream-сервисов; UX ID не используется как label.
- [ ] Все три сигнала поступают в выбранный бэкенд; SLO-запросы и оповещения проверены.
- [ ] При недоступности удаленного сервиса ограниченная постоянная очередь заполняется и опустошается без остановки бизнес-функций.
- [ ] Тестовые секреты и PII отсутствуют; проверены метрики перезапуска, потерь, отказов и очереди Collector.
Только для локальной приемки запустите отладочный Collector с удалением чувствительных данных:
`docker compose --profile observability-local up -d otel-collector-local`.
## Этап 16 — открытие трафика
- [ ] Этапы 0–15 подписаны; имеются свежие подтверждения backup/PITR и предыдущие образы.
- [ ] HSTS включен; дайджесты релиза, версии схем и realm зафиксированы.
- [ ] Активных инцидентов нет; дежурный и владелец продукта приняли ограничения заглушек.
- [ ] В течение 60 минут контролируются 5xx, auth, доставка, БД, Redis, OOM, очередь OTEL и Bitrix.
## Резервное копирование и восстановление
Основным механизмом являются backup/PITR провайдера. Дополнительный проверенный логический дамп:
```sh
deployment/scripts/backup.sh /opt/han-chat/backups
```
Ежеквартально восстанавливайте PostgreSQL и S3 в изолированной VPC, развертывайте те же
дайджесты образов, не направляйте туда production DNS и callbacks Bitrix, выполняйте
smoke-тесты и фиксируйте фактические RPO/RTO. Redis можно восстановить пустым:
его AOF/RDB не является резервной копией бизнес-данных.
## Откат
Откатывайтесь только на образы, совместимые с текущей схемой:
```sh
SCHEMA_BACKWARD_COMPATIBLE_CONFIRMED=true \
deployment/scripts/rollback.sh <PREVIOUS_IMMUTABLE_RELEASE>
deployment/scripts/smoke.sh
```
Откат использует текущие runtime-секреты и текущий несекретный config. Snapshot
старого `.env` не создаётся и не восстанавливается.
## Дополнение: rollout реальной SMS-авторизации
Текущий runbook остаётся mock-only, пока артефакты module-11 не реализованы. Для SMS release обязательны: schema/role `sms`, migrations/seed active approved `auth_otp`, `sms-service`/worker, exact callback route, парные service tokens, Direct `TOKEN_1`, согласованные sender/template, отдельные callback credentials, подтверждённый callback source IP и статический egress IP worker.
Порядок: App DB OTP seed → SMS schema/migrations/seed → test с mock Direct → production SMS deploy при `KEYCLOAK_OTP_MOCK_ENABLED=true` → Keycloak expand migration/SPI → provider smoke и callback/redaction evidence → real mode. Rollback: вернуть mock, не удалять journal/schema, остановить новые real orders и зафиксировать in-flight/`uncertain`; downgrade только при доказанной совместимости.
Никогда не выполняйте downgrade Alembic. После обратно несовместимой миграции используйте
исправление вперед либо согласованный PITR с восстановлением S3 и сверкой Bitrix во время
технического обслуживания. Всегда проверяйте outbox, inbox и recovery, чтобы сообщение
с неопределенным статусом не было отправлено повторно.
VM1_app/codebase/backend/deployment/app-settings.production-like.yaml
+51
View File
@@ -0,0 +1,51 @@
schema_version: 1
settings:
auth.phone.enabled: {type: boolean, value: true, public: true}
auth.password.enabled: {type: boolean, value: false, public: true}
otp.phone.max_send_attempts_per_24h: {type: integer, value: 3, public: false}
otp.phone.min_seconds_between_attempts: {type: integer, value: 30, public: false}
otp.phone.max_verify_attempts: {type: integer, value: 5, public: false}
otp.phone.code_length: {type: integer, value: 6, public: false}
otp.phone.ttl_seconds: {type: integer, value: 60, public: false}
otp.phone.sms_order_timeout_ms: {type: integer, value: 3000, public: false}
operator.call.phone: {type: string, value: "+74999591007", public: true}
consent.personal_data.required: {type: boolean, value: true, public: true}
consent.personal_data.document_url: {type: string, value: "https://www.han0107.ru/privacy/persdata-agree-mobile", public: true}
consent.personal_data.version: {type: string, value: "2026-06-10", public: true}
consent.privacy_policy.document_url: {type: string, value: "https://www.han0107.ru/privacy", public: true}
consent.user_agreement.required: {type: boolean, value: true, public: true}
consent.user_agreement.document_url: {type: string, value: "https://www.han0107.ru/user-agreement", public: true}
consent.user_agreement.version: {type: string, value: "2026-06-10", public: true}
consent.marketing.required: {type: boolean, value: false, public: true}
consent.marketing.document_url: {type: string, value: "https://www.han0107.ru/privacy/ads-agree", public: true}
consent.marketing.version: {type: string, value: "2026-06-10", public: true}
chat.message.max_length: {type: integer, value: 4000, public: true}
chat.attachments.allowed_extensions: {type: string_list, value: "jpg,jpeg,png,webp,heic,heif,pdf", public: true}
chat.attachments.allowed_mime_types: {type: string_list, value: "image/jpeg,image/png,image/webp,image/heic,image/heif,application/pdf", public: true}
chat.attachments.disallowed_extensions: {type: string_list, value: "svg,doc,docx,xls,xlsx,csv", public: false}
chat.attachments.max_size_mb: {type: integer, value: 5, public: true}
chat.attachments.storage: {type: string, value: selectel_s3, public: false}
chat.attachments.upload_mode: {type: string, value: presigned_put, public: false}
chat.attachments.safety_scan_required: {type: boolean, value: true, public: false}
chat.attachments.presigned_upload_ttl_seconds: {type: integer, value: 600, public: true}
rate_limit.message_send.per_user: {type: string, value: "30/minute", public: true}
rate_limit.message_send.per_dialog: {type: string, value: "20/minute", public: false}
rate_limit.download_url.per_user: {type: string, value: "60/hour", public: false}
rate_limit.public_endpoints.per_ip: {type: string, value: "60/minute", public: true}
rate_limit.login.per_ip: {type: string, value: "10/minute", public: true}
rate_limit.notifications_read.per_user: {type: string, value: "120/minute", public: false}
rate_limit.notifications_action.per_user: {type: string, value: "60/minute", public: false}
rate_limit.notification_upload.per_user: {type: string, value: "20/minute", public: false}
rate_limit.notifications_public.per_ip: {type: string, value: "60/minute", public: false}
notification.home.max_items: {type: integer, value: 7, public: false}
notification.center.max_items: {type: integer, value: 15, public: false}
notification.carousel.autoplay_enabled: {type: boolean, value: false, public: true}
notification.carousel.autoplay_interval_ms: {type: integer, value: 5000, public: true}
notification.hidden.default_ttl_days: {type: integer, value: 3, public: false}
notification.documents.max_files: {type: integer, value: 10, public: false}
notification.instruction.allowed_hosts: {type: string_list, value: "chat.example.ru", public: false}
notification.expire_job.run_at: {type: string, value: "00:01", public: false}
notification.upload_draft.ttl_days: {type: integer, value: 7, public: false}
ux.session.idle_timeout_minutes: {type: integer, value: 30, public: true}
security.cors.allowed_origins: {type: string_list, value: "https://chat.example.ru", public: false}
security.public_cache.max_age_seconds: {type: integer, value: 3600, public: false}
VM1_app/codebase/backend/deployment/docker-compose.jobs.yml
+147
View File
@@ -0,0 +1,147 @@
x-api-job-secrets: &api-job-secrets
- api_database_url
- api_redis_url
- api_redis_realtime_url
- message_safety_service_token
- bitrix_local_app_internal_token
- bitrix_api_inbox_token
- keycloak_settings_bridge_token
- selectel_s3_access_key
- selectel_s3_secret_key
- cursor_hmac_secret
x-api-job-environment: &api-job-environment
HAN_SECRET_VARS: >-
DATABASE_URL REDIS_URL REDIS_REALTIME_URL MESSAGE_SAFETY_SERVICE_TOKEN
BITRIX_LOCAL_APP_INTERNAL_TOKEN BITRIX_API_INBOX_TOKEN
KEYCLOAK_SETTINGS_BRIDGE_TOKEN SELECTEL_S3_ACCESS_KEY SELECTEL_S3_SECRET_KEY
CURSOR_HMAC_SECRET
DATABASE_URL_FILE: /run/secrets/api_database_url
REDIS_URL_FILE: /run/secrets/api_redis_url
REDIS_REALTIME_URL_FILE: /run/secrets/api_redis_realtime_url
MESSAGE_SAFETY_SERVICE_TOKEN_FILE: /run/secrets/message_safety_service_token
BITRIX_LOCAL_APP_INTERNAL_TOKEN_FILE: /run/secrets/bitrix_local_app_internal_token
BITRIX_API_INBOX_TOKEN_FILE: /run/secrets/bitrix_api_inbox_token
KEYCLOAK_SETTINGS_BRIDGE_TOKEN_FILE: /run/secrets/keycloak_settings_bridge_token
SELECTEL_S3_ACCESS_KEY_FILE: /run/secrets/selectel_s3_access_key
SELECTEL_S3_SECRET_KEY_FILE: /run/secrets/selectel_s3_secret_key
CURSOR_HMAC_SECRET_FILE: /run/secrets/cursor_hmac_secret
APP_ENV: ${APP_ENV:-production-like}
LOG_LEVEL: ${LOG_LEVEL:-INFO}
KEYCLOAK_PUBLIC_URL: ${KEYCLOAK_PUBLIC_URL}
KEYCLOAK_INTERNAL_URL: ${KEYCLOAK_INTERNAL_URL:-http://keycloak:8080/auth}
KEYCLOAK_REALM: ${KEYCLOAK_REALM:-han-chat}
KEYCLOAK_AUDIENCE: ${KEYCLOAK_AUDIENCE:-han-chat-api}
MESSAGE_SAFETY_URL: ${MESSAGE_SAFETY_URL}
MESSAGE_SAFETY_API_PREFIX: /internal/safety/v2
MESSAGE_SAFETY_CA_FILE: /run/config/message-safety-internal-ca.pem
MESSAGE_SAFETY_POST_TIMEOUT_SEC: ${MESSAGE_SAFETY_POST_TIMEOUT_SEC:-5}
MESSAGE_SAFETY_TASK_POLL_INTERVAL_SEC: ${MESSAGE_SAFETY_TASK_POLL_INTERVAL_SEC:-2}
MESSAGE_SAFETY_TASK_POLL_MAX_SEC: ${MESSAGE_SAFETY_TASK_POLL_MAX_SEC:-300}
BITRIX_LOCAL_APP_BASE_URL: ${BITRIX_LOCAL_APP_BASE_URL:-http://bitrix-local-app:8080}
BITRIX_LOCAL_APP_HTTP_TIMEOUT_SEC: ${BITRIX_LOCAL_APP_HTTP_TIMEOUT_SEC:-20}
SELECTEL_S3_ENDPOINT_URL: ${SELECTEL_S3_ENDPOINT_URL}
SELECTEL_S3_BUCKET_DOCUMENTS: ${SELECTEL_S3_BUCKET_DOCUMENTS}
SELECTEL_S3_BUCKET_ATTACHMENTS: ${SELECTEL_S3_BUCKET_ATTACHMENTS}
SELECTEL_S3_BUCKET_QUARANTINE: ${SELECTEL_S3_BUCKET_QUARANTINE}
TRUSTED_PROXY_CIDRS: ${TRUSTED_PROXY_CIDRS:-127.0.0.1/32}
services:
migrate-api:
image: ${API_BACKEND_IMAGE:-han-chat-api-backend:local}
profiles: ["ops"]
environment:
HAN_SECRET_VARS: DATABASE_URL
DATABASE_URL_FILE: /run/secrets/api_database_url
secrets:
- api_database_url
command: ["alembic", "upgrade", "head"]
volumes:
- ${PG_CA_HOST_PATH}:/run/secrets/pg-ca.pem:ro
networks: [backend, egress]
restart: "no"
security_opt: ["no-new-privileges:true"]
ulimits:
core: {soft: 0, hard: 0}
migrate-bitrix-local:
image: ${BITRIX_LOCAL_APP_IMAGE:-han-chat-bitrix-local-app:local}
profiles: ["ops"]
environment:
HAN_SECRET_VARS: BITRIX_DATABASE_URL
BITRIX_DATABASE_URL_FILE: /run/secrets/bitrix_database_url
secrets:
- bitrix_database_url
command: ["alembic", "upgrade", "head"]
volumes:
- ${PG_CA_HOST_PATH}:/run/secrets/pg-ca.pem:ro
networks: [backend, egress]
restart: "no"
security_opt: ["no-new-privileges:true"]
ulimits:
core: {soft: 0, hard: 0}
migrate-bitrix-sync:
image: ${BITRIX_SYNC_IMAGE:-han-chat-bitrix-sync:local}
profiles: ["ops"]
environment:
HAN_SECRET_VARS: BITRIX_SYNC_DATABASE_URL
BITRIX_SYNC_DATABASE_URL_FILE: /run/secrets/bitrix_sync_database_url
secrets:
- bitrix_sync_database_url
command: ["alembic", "upgrade", "head"]
volumes:
- ${PG_CA_HOST_PATH}:/run/secrets/pg-ca.pem:ro
networks: [backend, egress]
restart: "no"
security_opt: ["no-new-privileges:true"]
ulimits:
core: {soft: 0, hard: 0}
migrate-sms:
image: ${SMS_SERVICE_IMAGE:-han-chat-sms-service:local}
profiles: ["ops"]
environment:
HAN_SECRET_VARS: SMS_DATABASE_URL
SMS_DATABASE_URL_FILE: /run/secrets/sms_database_url
secrets:
- sms_database_url
command: ["alembic", "upgrade", "head"]
volumes:
- ${PG_CA_HOST_PATH}:/run/secrets/pg-ca.pem:ro
networks: [backend, egress]
restart: "no"
security_opt: ["no-new-privileges:true"]
ulimits:
core: {soft: 0, hard: 0}
seed-settings:
image: ${API_BACKEND_IMAGE:-han-chat-api-backend:local}
profiles: ["ops"]
environment: *api-job-environment
secrets: *api-job-secrets
command:
- /bin/sh
- -ec
- >-
python -m app.cli.seed_settings
--file /deployment/app-settings.production-like.yaml
&& python -m app.cli.validate_settings
volumes:
- ${PG_CA_HOST_PATH}:/run/secrets/pg-ca.pem:ro
- ${MESSAGE_SAFETY_CA_HOST_PATH}:/run/config/message-safety-internal-ca.pem:ro
- ./app-settings.production-like.yaml:/deployment/app-settings.production-like.yaml:ro
networks: [backend, egress]
restart: "no"
security_opt: ["no-new-privileges:true"]
ulimits:
core: {soft: 0, hard: 0}
toolbox:
image: curlimages/curl:8.11.1
profiles: ["ops"]
entrypoint: ["sleep", "infinity"]
networks: [backend, observability, egress]
restart: "no"
cap_drop: ["ALL"]
security_opt: ["no-new-privileges:true"]
VM1_app/codebase/backend/deployment/scripts/backup.sh
+38
View File
@@ -0,0 +1,38 @@
#!/bin/sh
set -eu
cd "$(dirname "$0")/../.."
CONFIG_FILE=${CONFIG_FILE:-.env}
SECRETS_LAUNCHER=${SECRETS_LAUNCHER:-deployment/secrets/han-secrets}
if [ "${HAN_SECRETS_ACTIVE:-0}" != "1" ]; then
[ -x "$SECRETS_LAUNCHER" ] || {
echo "Secret launcher is required: $SECRETS_LAUNCHER" >&2
exit 66
}
exec "$SECRETS_LAUNCHER" run --config "$CONFIG_FILE" -- "$0" "$@"
fi
if [ -z "${PG_BACKUP_DSN_FILE:-}" ] || [ ! -r "$PG_BACKUP_DSN_FILE" ]; then
echo "PG_BACKUP_DSN_FILE is required from the secret launcher." >&2
exit 64
fi
PG_BACKUP_DSN=$(cat "$PG_BACKUP_DSN_FILE")
case "$PG_BACKUP_DSN" in
*sslmode=verify-full*sslrootcert=*) ;;
*) echo "PG_BACKUP_DSN must enforce sslmode=verify-full and sslrootcert." >&2; exit 64 ;;
esac
output_dir=${1:-/opt/han-chat/backups}
umask 077
mkdir -p "$output_dir"
stamp=$(date -u +%Y%m%dT%H%M%SZ)
archive="$output_dir/han-chat-$stamp.dump"
PGDATABASE="$PG_BACKUP_DSN" pg_dump \
--format=custom --no-owner --no-privileges --file="$archive"
unset PG_BACKUP_DSN
pg_restore --list "$archive" >/dev/null
sha256sum "$archive" > "$archive.sha256"
chmod 600 "$archive" "$archive.sha256"
echo "Logical backup verified: $archive"
echo "This supplements, but does not replace, provider backup/PITR and restore rehearsal."
VM1_app/codebase/backend/deployment/scripts/migrate.sh
+34
View File
@@ -0,0 +1,34 @@
#!/bin/sh
set -eu
cd "$(dirname "$0")/../.."
CONFIG_FILE=${CONFIG_FILE:-.env}
SECRETS_LAUNCHER=${SECRETS_LAUNCHER:-deployment/secrets/han-secrets}
if [ "${HAN_SECRETS_ACTIVE:-0}" != "1" ]; then
[ -x "$SECRETS_LAUNCHER" ] || {
echo "Secret launcher is required: $SECRETS_LAUNCHER" >&2
exit 66
}
exec "$SECRETS_LAUNCHER" run --config "$CONFIG_FILE" -- "$0" "$@"
fi
if [ "${PITR_MARKER_CONFIRMED:-false}" != "true" ]; then
echo "Refusing migration: create provider PITR marker, then set PITR_MARKER_CONFIRMED=true" >&2
exit 64
fi
if [ -n "${HAN_RUNTIME_SECRET_MANIFEST:-}" ]; then
./scripts/validate-env "$CONFIG_FILE" --runtime-manifest "$HAN_RUNTIME_SECRET_MANIFEST"
else
./scripts/validate-env "$CONFIG_FILE" --runtime-env
fi
docker compose --env-file "$CONFIG_FILE" config --quiet
docker compose --env-file "$CONFIG_FILE" --profile ops run --rm migrate-api alembic current
docker compose --env-file "$CONFIG_FILE" --profile ops run --rm migrate-bitrix-local alembic current
docker compose --env-file "$CONFIG_FILE" --profile ops run --rm migrate-bitrix-sync alembic current
docker compose --env-file "$CONFIG_FILE" --profile ops run --rm migrate-sms alembic current
docker compose --env-file "$CONFIG_FILE" --profile ops run --rm migrate-api
docker compose --env-file "$CONFIG_FILE" --profile ops run --rm migrate-bitrix-local
docker compose --env-file "$CONFIG_FILE" --profile ops run --rm migrate-bitrix-sync
docker compose --env-file "$CONFIG_FILE" --profile ops run --rm migrate-sms
echo "Migrations completed; record revisions in release evidence."
VM1_app/codebase/backend/deployment/scripts/rollback.sh
+34
View File
@@ -0,0 +1,34 @@
#!/bin/sh
set -eu
cd "$(dirname "$0")/../.."
CONFIG_FILE=${CONFIG_FILE:-.env}
SECRETS_LAUNCHER=${SECRETS_LAUNCHER:-deployment/secrets/han-secrets}
previous_release=${1:-}
if [ -z "$previous_release" ]; then
echo "Usage: $0 <previous-immutable-release>" >&2
exit 64
fi
if [ "${HAN_SECRETS_ACTIVE:-0}" != "1" ]; then
[ -x "$SECRETS_LAUNCHER" ] || {
echo "Secret launcher is required: $SECRETS_LAUNCHER" >&2
exit 66
}
exec "$SECRETS_LAUNCHER" run --config "$CONFIG_FILE" -- "$0" "$@"
fi
if [ "${SCHEMA_BACKWARD_COMPATIBLE_CONFIRMED:-false}" != "true" ]; then
echo "Refusing rollback: set SCHEMA_BACKWARD_COMPATIBLE_CONFIRMED=true after migration review." >&2
exit 64
fi
if [ -n "${HAN_RUNTIME_SECRET_MANIFEST:-}" ]; then
./scripts/validate-env "$CONFIG_FILE" --runtime-manifest "$HAN_RUNTIME_SECRET_MANIFEST"
else
./scripts/validate-env "$CONFIG_FILE" --runtime-env
fi
RELEASE_VERSION="$previous_release" docker compose --env-file "$CONFIG_FILE" config --quiet
RELEASE_VERSION="$previous_release" docker compose --env-file "$CONFIG_FILE" up -d --remove-orphans
docker compose --env-file "$CONFIG_FILE" ps
echo "Application images rolled back without Alembic downgrade."
echo "Run deployment/scripts/smoke.sh and verify outbox/inbox idempotency."
VM1_app/codebase/backend/deployment/scripts/seed-personal-notifications-test.sh
+448
View File
@@ -0,0 +1,448 @@
#!/usr/bin/env bash
# Создаёт 9 персональных уведомлений всех видов контура P для одного user_id.
# Запускать на ВМ из каталога backend: /opt/han-chat/backend
#
# cd /opt/han-chat/backend
# sed -i 's/\r$//' deployment/scripts/seed-personal-notifications-test.sh
# chmod +x deployment/scripts/seed-personal-notifications-test.sh
# ./deployment/scripts/seed-personal-notifications-test.sh
#
# Токен берётся из .env (NOTIFICATIONS_TOKEN_PRODUCER_TEST) — тот же, что у api-backend.
# При старте api-backend синхронизирует hash токена в notification_sources.
set -euo pipefail
cd "$(dirname "$0")/../.."
ENV_FILE="${ENV_FILE:-.env}"
if [[ ! -f "$ENV_FILE" ]]; then
echo "Не найден $ENV_FILE. Запускайте из каталога backend." >&2
exit 1
fi
env_value() {
python3 - "$ENV_FILE" "$1" <<'PY'
import sys
from pathlib import Path
path, wanted = sys.argv[1:]
for raw in Path(path).read_text(encoding="utf-8").splitlines():
line = raw.strip()
if not line or line.startswith("#") or "=" not in line:
continue
key, value = line.split("=", 1)
if key.strip() == wanted:
value = value.strip()
if len(value) >= 2 and value[0] == value[-1] and value[0] in "\"'":
value = value[1:-1]
print(value)
break
else:
raise SystemExit(f"missing environment variable: {wanted}")
PY
}
trim_token() {
printf '%s' "$1" | tr -d '\r\n\t '
}
TOKEN="$(trim_token "${NOTIFICATIONS_TOKEN_PRODUCER_TEST:-}")"
if [[ -z "$TOKEN" ]]; then
TOKEN="$(trim_token "$(env_value NOTIFICATIONS_TOKEN_PRODUCER_TEST 2>/dev/null || true)")"
fi
if [[ -z "$TOKEN" ]]; then
read -rsp "NOTIFICATIONS_TOKEN_PRODUCER_TEST (из .env не найден): " TOKEN
echo
TOKEN="$(trim_token "$TOKEN")"
fi
read -rp "USER_ID клиента: " USER_ID
USER_ID="$(trim_token "$USER_ID")"
if [[ -z "${TOKEN}" || -z "${USER_ID}" ]]; then
echo "TOKEN и USER_ID обязательны." >&2
exit 1
fi
echo "Токен: ${#TOKEN} символов (первые 8: ${TOKEN:0:8}…)"
PAYLOAD_DIR="$(mktemp -d)"
trap 'rm -rf "$PAYLOAD_DIR"' EXIT
BASE_TS="$(date +%s)"
RUN_ID="manual-test-${BASE_TS}"
NOW="$(date -u +"%Y-%m-%dT%H:%M:%SZ")"
DT_URGENT="$(date -u -d "${NOW} +0 seconds" +"%Y-%m-%dT%H:%M:%SZ" 2>/dev/null || date -u -v+0S +"%Y-%m-%dT%H:%M:%SZ")"
DT_PAYMENT="$(date -u -d "${NOW} - 60 seconds" +"%Y-%m-%dT%H:%M:%SZ" 2>/dev/null || date -u -v-60S +"%Y-%m-%dT%H:%M:%SZ")"
DT_DOCS_REQ="$(date -u -d "${NOW} - 120 seconds" +"%Y-%m-%dT%H:%M:%SZ" 2>/dev/null || date -u -v-120S +"%Y-%m-%dT%H:%M:%SZ")"
DT_DOCS_READY="$(date -u -d "${NOW} - 180 seconds" +"%Y-%m-%dT%H:%M:%SZ" 2>/dev/null || date -u -v-180S +"%Y-%m-%dT%H:%M:%SZ")"
DT_STATUS="$(date -u -d "${NOW} - 240 seconds" +"%Y-%m-%dT%H:%M:%SZ" 2>/dev/null || date -u -v-240S +"%Y-%m-%dT%H:%M:%SZ")"
DT_REMINDER="$(date -u -d "${NOW} - 300 seconds" +"%Y-%m-%dT%H:%M:%SZ" 2>/dev/null || date -u -v-300S +"%Y-%m-%dT%H:%M:%SZ")"
DT_NEWS="$(date -u -d "${NOW} - 360 seconds" +"%Y-%m-%dT%H:%M:%SZ" 2>/dev/null || date -u -v-360S +"%Y-%m-%dT%H:%M:%SZ")"
DT_PROMO="$(date -u -d "${NOW} - 420 seconds" +"%Y-%m-%dT%H:%M:%SZ" 2>/dev/null || date -u -v-420S +"%Y-%m-%dT%H:%M:%SZ")"
DT_ADS="$(date -u -d "${NOW} - 480 seconds" +"%Y-%m-%dT%H:%M:%SZ" 2>/dev/null || date -u -v-480S +"%Y-%m-%dT%H:%M:%SZ")"
DEADLINE_URGENT="$(date -u -d "${NOW} + 2 days" +"%Y-%m-%dT%H:%M:%SZ" 2>/dev/null || date -u -v+2d +"%Y-%m-%dT%H:%M:%SZ")"
DEADLINE_DOCS="$(date -u -d "${NOW} + 5 days" +"%Y-%m-%dT%H:%M:%SZ" 2>/dev/null || date -u -v+5d +"%Y-%m-%dT%H:%M:%SZ")"
DEADLINE_REMINDER="$(date -u -d "${NOW} + 1 day" +"%Y-%m-%dT%H:%M:%SZ" 2>/dev/null || date -u -v+1d +"%Y-%m-%dT%H:%M:%SZ")"
EXPIRE_PAYMENT="$(date -u -d "${NOW} + 3 days" +"%Y-%m-%dT%H:%M:%SZ" 2>/dev/null || date -u -v+3d +"%Y-%m-%dT%H:%M:%SZ")"
write_payload() {
local name="$1"
cat > "${PAYLOAD_DIR}/${name}.json"
}
create_notification() {
local label="$1"
local payload_file="${PAYLOAD_DIR}/${label}.json"
if [[ ! -f "$payload_file" ]]; then
echo "Нет payload: ${payload_file}" >&2
return 1
fi
echo
echo "========== ${label} =========="
docker run --rm \
--network han-chat-backend \
-v "${payload_file}:/payload.json:ro" \
curlimages/curl:latest \
-sS -i \
-X POST \
'http://api-backend:8000/internal/notifications/v1/notifications' \
-H "Authorization: Bearer ${TOKEN}" \
-H 'Content-Type: application/json; charset=utf-8' \
--data-binary @/payload.json
}
prepare_docs_in_s3() {
echo >&2
echo "========== PREP: загрузка тестовых документов в S3 для docs_ready ==========" >&2
docker compose --env-file "$ENV_FILE" exec -T \
-e "USER_ID=${USER_ID}" \
-e "RUN_ID=${RUN_ID}" \
api-backend python3 - <<'PY'
import hashlib
import os
import uuid
import boto3
from botocore.client import Config
from app.settings import Settings
USER_ID = os.environ["USER_ID"]
settings = Settings()
client = boto3.client(
"s3",
endpoint_url=str(settings.selectel_s3_endpoint_url),
aws_access_key_id=settings.selectel_s3_access_key.get_secret_value(),
aws_secret_access_key=settings.selectel_s3_secret_key.get_secret_value(),
config=Config(signature_version="s3v4", s3={"addressing_style": "virtual"}),
)
bucket = settings.selectel_s3_bucket_documents
fixtures = [
{
"prefix": "CONTRACT",
"title": "Уведомление о постановке на миграционный учёт.pdf",
"body": b"""%PDF-1.4
1 0 obj<</Type/Catalog/Pages 2 0 R>>endobj
2 0 obj<</Type/Pages/Kids[3 0 R]/Count 1>>endobj
3 0 obj<</Type/Page/MediaBox[0 0 612 792]/Parent 2 0 R>>endobj
xref
0 4
trailer<</Size 4/Root 1 0 R>>
startxref
100
%%EOF
""",
},
{
"prefix": "RESULTS",
"title": "Справка о соблюдении миграционного законодательства.pdf",
"body": b"""%PDF-1.4
1 0 obj<</Type/Catalog/Pages 2 0 R>>endobj
2 0 obj<</Type/Pages/Kids[3 0 R]/Count 1>>endobj
3 0 obj<</Type/Page/MediaBox[0 0 612 792]/Parent 2 0 R>>endobj
xref
0 4
trailer<</Size 4/Root 1 0 R>>
startxref
100
%%EOF
""",
},
]
lines: list[str] = []
for item in fixtures:
doc_id = str(uuid.uuid4())
key = f"documents/users/{USER_ID}/{doc_id}"
body = item["body"]
checksum = hashlib.sha256(body).hexdigest()
client.put_object(
Bucket=bucket,
Key=key,
Body=body,
ContentType="application/pdf",
)
p = item["prefix"]
lines.append(
f"{p}_KEY={key}\n"
f"{p}_TITLE={item['title']}\n"
f"{p}_SIZE={len(body)}\n"
f"{p}_SHA256={checksum}"
)
print("\n".join(lines))
PY
}
echo "Run ID: ${RUN_ID}"
echo "User ID: ${USER_ID}"
read -rp "Подготовить документы в S3 для docs_ready? [Y/n]: " PREP_DOCS
PREP_DOCS="${PREP_DOCS:-Y}"
CONTRACT_KEY=""
CONTRACT_TITLE=""
CONTRACT_SIZE=""
CONTRACT_SHA256=""
RESULTS_KEY=""
RESULTS_TITLE=""
RESULTS_SIZE=""
RESULTS_SHA256=""
if [[ "${PREP_DOCS^^}" != "N" ]]; then
PREP_OUT="$(prepare_docs_in_s3)"
echo "${PREP_OUT}"
while IFS= read -r line; do
[[ "$line" =~ ^([A-Z][A-Z0-9_]*)=(.*)$ ]] || continue
declare "${BASH_REMATCH[1]}=${BASH_REMATCH[2]}"
done <<< "${PREP_OUT}"
fi
# --- payloads ---
write_payload urgent <<JSON
{
"user_id": "${USER_ID}",
"notification_type": "urgent",
"source": "producer_test",
"external_id": "${RUN_ID}-urgent",
"notification_datetime": "${DT_URGENT}",
"header": "Истекает срок постановки на учёт",
"text": "По данным сервиса, срок уведомления о месте пребывания истекает 29 июля — просрочка влечёт административную ответственность.",
"priority_override": 1,
"date_expired": "${DEADLINE_URGENT}",
"details": {
"deadline": "${DEADLINE_URGENT}",
"details_header": "Срочно: соблюдение сроков миграционного учёта",
"details_text": "Федеральный закон № 109-ФЗ обязывает иностранного гражданина в течение 7 рабочих дней с даты въезда подать уведомление о прибытии в место пребывания (если иное не предусмотрено для вашего правового статуса).\n\nПо имеющимся данным крайний срок для вашего случая — 29.07.2026. Нарушение сроков может повлечь штраф от 2 000 до 5 000 ₽ и, при повторном нарушении, более серьёзные последствия, включая административное выдворение.\n\nЕсли уведомление уже подано — отметьте «Готово»; если нужна помощь — напишите оператору в чат.",
"todo_header": "Что проверить сейчас",
"todo_plan": [
{"number": 1, "text": "Сверьте дату въезда и адрес фактического проживания в анкете личного кабинета."},
{"number": 2, "text": "Подготовьте копии паспорта, миграционной карты и документа о праве пребывания (виза, РВП, ВНЖ, патент)."},
{"number": 3, "text": "Нажмите «Готово», если уведомление уже подано через МВД или принимающую сторону."},
{"number": 4, "text": "При сомнениях выберите «Сделаю позже» и задайте вопрос оператору — укажите город и тип документа."}
]
}
}
JSON
write_payload payment_pending <<JSON
{
"user_id": "${USER_ID}",
"notification_type": "payment_pending",
"source": "producer_test",
"external_id": "${RUN_ID}-payment_pending",
"notification_datetime": "${DT_PAYMENT}",
"header": "Оплата сопровождения по миграционному учёту",
"text": "Счёт №МИГ-2026-0718 на 18 500 ₽ за подготовку пакета и подачу уведомления — оплатите до 30 июля.",
"price": "18500.00",
"old_price": "22000.00",
"date_expired": "${EXPIRE_PAYMENT}",
"payment_url": "https://pay.han0107.ru/checkout/test-${RUN_ID}"
}
JSON
write_payload docs_required <<JSON
{
"user_id": "${USER_ID}",
"notification_type": "docs_required",
"source": "producer_test",
"external_id": "${RUN_ID}-docs_required",
"notification_datetime": "${DT_DOCS_REQ}",
"header": "Загрузите документы для миграционного учёта",
"text": "Для проверки соблюдения миграционного законодательства нужен комплект документов — загрузите до 01 августа.",
"date_expired": "${DEADLINE_DOCS}",
"details": {
"deadline": "${DEADLINE_DOCS}",
"details_header": "Документы для постановки на миграционный учёт",
"details_text": "Специалист проверит комплект в течение 1 рабочего дня после отправки. Принимаются чёткие фото или сканы в JPG, PNG, PDF. Каждый файл — до 10 МБ, не более 10 файлов за одну отправку.\n\nВсе документы должны быть действительными на дату проверки. Если какого-то документа пока нет (например, договор найма ещё не подписан), загрузите доступные — оператор подскажет порядок действий и допустимые альтернативы.",
"todo_header": "Необходимый пакет",
"todo_plan": [
{"number": 1, "text": "Паспорт: страница с фото, действующая виза или иной документ на право пребывания."},
{"number": 2, "text": "Миграционная карта с отметкой о въезде (обе стороны)."},
{"number": 3, "text": "Документ о месте пребывания: договор найма, свидетельство собственности или письмо принимающей стороны."},
{"number": 4, "text": "При трудовой деятельности — копия патента или разрешения на работу (если применимо)."},
{"number": 5, "text": "Нажмите «Отправить документы», когда все файлы приложены."}
],
"send_documents": true
}
}
JSON
if [[ -n "${CONTRACT_KEY}" && -n "${RESULTS_KEY}" ]]; then
write_payload docs_ready <<JSON
{
"user_id": "${USER_ID}",
"notification_type": "docs_ready",
"source": "producer_test",
"external_id": "${RUN_ID}-docs_ready",
"notification_datetime": "${DT_DOCS_READY}",
"header": "Миграционные документы готовы",
"text": "Уведомление о постановке на учёт и справка о соблюдении требований закона сформированы — скачайте в деталях.",
"details": {
"details_header": "Документы по миграционному учёту",
"details_text": "Документы подготовлены на основании переданных вами данных и проверены специалистом. Сохраните копии на устройство — они могут понадобиться при проверке или продлении статуса пребывания.\n\nСсылки на скачивание действуют ограниченное время. После первого скачивания карточка скроется с главной, но останется в Центре уведомлений до нажатия «Понятно».",
"documents": [
{
"object_key": "${CONTRACT_KEY}",
"title": "${CONTRACT_TITLE}",
"mime_type": "application/pdf",
"size_bytes": ${CONTRACT_SIZE},
"checksum_sha256": "${CONTRACT_SHA256}"
},
{
"object_key": "${RESULTS_KEY}",
"title": "${RESULTS_TITLE}",
"mime_type": "application/pdf",
"size_bytes": ${RESULTS_SIZE},
"checksum_sha256": "${RESULTS_SHA256}"
}
]
}
}
JSON
fi
write_payload status_changed <<JSON
{
"user_id": "${USER_ID}",
"notification_type": "status_changed",
"source": "producer_test",
"external_id": "${RUN_ID}-status_changed",
"notification_datetime": "${DT_STATUS}",
"header": "Статус миграционной заявки обновлён",
"text": "Заявка №МГ-8841 переведена на этап «Проверка комплектности документов».",
"details": {
"details_header": "Ход рассмотрения заявки №МГ-8841",
"details_text": "27.07.2026 в 11:42 специалист принял ваш пакет документов в работу. Сейчас выполняется проверка на соответствие требованиям миграционного законодательства РФ: сроки пребывания, адрес учёта, основания для трудовой деятельности.\n\nОжидаемое время на этом этапе: 1–2 рабочих дня. При выявлении недостающих документов вы получите отдельное уведомление с перечнем. При успешной проверке будет сформировано уведомление о постановке на учёт.",
"todo_header": "Этапы обработки",
"todo_plan": [
{"number": 1, "text": "Документы получены — выполнено 25.07.2026"},
{"number": 2, "text": "Первичная верификация — выполнено 27.07.2026"},
{"number": 3, "text": "Проверка комплектности и сроков — в работе"},
{"number": 4, "text": "Формирование уведомления / рекомендаций — ожидает"}
]
}
}
JSON
write_payload reminder <<JSON
{
"user_id": "${USER_ID}",
"notification_type": "reminder",
"source": "producer_test",
"external_id": "${RUN_ID}-reminder",
"notification_datetime": "${DT_REMINDER}",
"header": "Напоминание: продление патента",
"text": "28 июля истекает срок действия патента — подайте заявление на продление заранее.",
"date_expired": "${DEADLINE_REMINDER}",
"details": {
"deadline": "${DEADLINE_REMINDER}",
"details_header": "Сроки продления документа на право работы",
"details_text": "Патент на работу необходимо продлевать заблаговременно — подача заявления рекомендуется не позднее чем за 10–15 рабочих дней до даты окончания действия. Просрочка означает прекращение права на трудовую деятельность и риск штрафа по ст. 18.15 КоАП РФ.\n\nДля продления потребуются: действующий патент, чеки об оплате авансовых платежей по НДФЛ, полис ДМС, сертификат о знании русского языка (если срок действия истекает), договор найма или иной документ о месте пребывания.",
"todo_header": "Чек-лист перед подачей",
"todo_plan": [
{"number": 1, "text": "Проверьте дату окончания патента в личном кабинете или на бланке документа."},
{"number": 2, "text": "Убедитесь, что авансовые платежи по НДФЛ оплачены без просрочки."},
{"number": 3, "text": "Подготовьте сканы документов — при необходимости загрузите через уведомление «Требуются документы»."},
{"number": 4, "text": "При вопросах напишите оператору — укажите регион и номер патента."}
]
}
}
JSON
write_payload news <<JSON
{
"user_id": "${USER_ID}",
"notification_type": "news",
"source": "producer_test",
"external_id": "${RUN_ID}-news",
"notification_datetime": "${DT_NEWS}",
"header": "Изменения в правилах миграционного учёта",
"text": "С 1 августа 2026 уточнены сроки подачи уведомлений при смене адреса пребывания.",
"details": {
"details_header": "Что изменилось для иностранных граждан",
"details_text": "С 01.08.2026 при смене адреса фактического проживания в том же субъекте РФ уведомление необходимо подать в течение 3 рабочих дней (ранее — 7). При переезде в другой регион срок остаётся 7 рабочих дней с даты регистрации по новому адресу.\n\nСервис HAN напомнит о приближающихся сроках через Центр уведомлений. Рекомендуем заранее подготовить копии договора найма и отметку о регистрации — это ускорит проверку оператором.\n\nПодробности процедуры — в чате с оператором или на официальном портале МВД России."
}
}
JSON
write_payload promo_personal <<JSON
{
"user_id": "${USER_ID}",
"notification_type": "promo_personal",
"source": "producer_test",
"external_id": "${RUN_ID}-promo_personal",
"notification_datetime": "${DT_PROMO}",
"header": "Скидка 15% на годовое сопровождение",
"text": "Персональное предложение: контроль сроков патента, учёта и уведомлений — до конца месяца.",
"price": "15725.00",
"old_price": "18500.00",
"date_expired": "${EXPIRE_PAYMENT}",
"chat_message_text": "Здравствуйте! Хочу подключить годовое сопровождение по соблюдению миграционного законодательства со скидкой 15%. Подскажите, что входит в пакет и как оформить."
}
JSON
write_payload ads_personal <<JSON
{
"user_id": "${USER_ID}",
"notification_type": "ads_personal",
"source": "producer_test",
"external_id": "${RUN_ID}-ads_personal",
"notification_datetime": "${DT_ADS}",
"header": "Бесплатная проверка миграционного статуса",
"text": "15 минут с экспертом: оценим риски и составим чек-лист обязательных действий.",
"chat_message_text": "Здравствуйте! Хочу воспользоваться бесплатной проверкой миграционного статуса. Подскажите, как записаться и какие документы подготовить к консультации."
}
JSON
# --- отправка ---
create_notification urgent
create_notification payment_pending
create_notification docs_required
if [[ -f "${PAYLOAD_DIR}/docs_ready.json" ]]; then
create_notification docs_ready
else
echo
echo "========== docs_ready — ПРОПУЩЕН =========="
fi
create_notification status_changed
create_notification reminder
create_notification news
create_notification promo_personal
create_notification ads_personal
echo
echo "========== Готово =========="
echo "External ID prefix: ${RUN_ID}-*"
echo
echo "Если видите 401: проверьте NOTIFICATIONS_TOKEN_PRODUCER_TEST в .env и перезапустите api-backend."
echo " grep NOTIFICATIONS_TOKEN_PRODUCER_TEST .env"
echo " docker compose --env-file .env up -d --force-recreate api-backend"
VM1_app/codebase/backend/deployment/scripts/seed.sh
+18
View File
@@ -0,0 +1,18 @@
#!/bin/sh
set -eu
cd "$(dirname "$0")/../.."
CONFIG_FILE=${ENV_FILE:-.env}
SECRETS_LAUNCHER=${SECRETS_LAUNCHER:-deployment/secrets/han-secrets}
if [ "${HAN_SECRETS_ACTIVE:-0}" != "1" ]; then
[ -x "$SECRETS_LAUNCHER" ] || {
echo "Secret launcher is required: $SECRETS_LAUNCHER" >&2
exit 66
}
exec "$SECRETS_LAUNCHER" run --config "$CONFIG_FILE" -- "$0" "$@"
fi
./scripts/validate-env "$CONFIG_FILE" \
--runtime-manifest "$HAN_RUNTIME_SECRET_MANIFEST"
docker compose --env-file "$CONFIG_FILE" --profile ops run --rm seed-settings
echo "Seed and mandatory-settings validation completed."
VM1_app/codebase/backend/deployment/scripts/setup-vm.sh
+591
View File
@@ -0,0 +1,591 @@
#!/usr/bin/env bash
# Первичная подготовка Ubuntu 24.04 для HAN Chat.
#
# Скрипт настраивает только VM: пользователя развертывания, базовые пакеты,
# Docker/Compose, UFW, fail2ban, DOCKER-USER, swap и каталоги проекта.
# PostgreSQL и S3 остаются внешними управляемыми сервисами. Скрипт не создает
# .env, секреты, DNS, S3-бакеты, схемы БД и TLS-сертификаты.
#
# Запуск на свежей VM:
# chmod +x deployment/scripts/setup-vm.sh
# sudo deployment/scripts/setup-vm.sh
#
# Основные параметры:
# DEPLOY_USER=deploy
# DEPLOY_DIR=/opt/han-chat/backend
# SSH_PORT=22
# TIMEZONE=Europe/Moscow
# SWAP_SIZE_GB=4
# EXTERNAL_IF=ens3
# PUBLIC_DOCKER_PORTS=80,443
# COPY_SSH_KEYS=true
# HARDEN_SSH=false
# LOCK_ACCOUNT_PASSWORDS=true
# HSTS_MAX_AGE_SECONDS=31536000
# RESET_UFW=false
# SKIP_APT_UPGRADE=false
#
# Парольный SSH-вход, X11 forwarding и локальные пароли root/deploy отключаются
# по умолчанию после проверки authorized_keys. HARDEN_SSH=true дополнительно
# запрещает прямой root-вход и SSH TCP forwarding.
set -Eeuo pipefail
IFS=$'\n\t'
DEPLOY_USER="${DEPLOY_USER:-deploy}"
DEPLOY_DIR="${DEPLOY_DIR:-/opt/han-chat/backend}"
SSH_PORT="${SSH_PORT:-22}"
TIMEZONE="${TIMEZONE:-Europe/Moscow}"
SWAP_SIZE_GB="${SWAP_SIZE_GB:-4}"
EXTERNAL_IF="${EXTERNAL_IF:-}"
PUBLIC_DOCKER_PORTS="${PUBLIC_DOCKER_PORTS:-80,443}"
COPY_SSH_KEYS="${COPY_SSH_KEYS:-true}"
HARDEN_SSH="${HARDEN_SSH:-false}"
LOCK_ACCOUNT_PASSWORDS="${LOCK_ACCOUNT_PASSWORDS:-true}"
HSTS_MAX_AGE_SECONDS="${HSTS_MAX_AGE_SECONDS:-31536000}"
RESET_UFW="${RESET_UFW:-false}"
SKIP_APT_UPGRADE="${SKIP_APT_UPGRADE:-false}"
LOG_FILE="${LOG_FILE:-/var/log/han-chat-vm-setup.log}"
log() {
printf '[%s] %s\n' "$(date '+%Y-%m-%d %H:%M:%S')" "$*" | tee -a "$LOG_FILE"
}
step() {
log ""
log "==> $*"
}
die() {
log "ОШИБКА: $*"
exit 1
}
on_error() {
local exit_code=$?
log "ОШИБКА: команда завершилась с кодом ${exit_code}, строка ${BASH_LINENO[0]}"
exit "$exit_code"
}
trap on_error ERR
require_root() {
[[ "${EUID:-$(id -u)}" -eq 0 ]] || die "Запустите скрипт через sudo"
}
validate_parameters() {
[[ "$DEPLOY_USER" =~ ^[a-z_][a-z0-9_-]*$ ]] || die "Некорректный DEPLOY_USER"
[[ "$DEPLOY_DIR" == /* ]] || die "DEPLOY_DIR должен быть абсолютным путем"
[[ "$SSH_PORT" =~ ^[0-9]+$ ]] || die "SSH_PORT должен быть числом"
((SSH_PORT >= 1 && SSH_PORT <= 65535)) || die "SSH_PORT вне диапазона"
[[ "$SWAP_SIZE_GB" =~ ^[0-9]+$ ]] || die "SWAP_SIZE_GB должен быть целым числом"
[[ "$HSTS_MAX_AGE_SECONDS" =~ ^[0-9]+$ ]] \
|| die "HSTS_MAX_AGE_SECONDS должен быть целым числом"
((HSTS_MAX_AGE_SECONDS >= 31536000)) \
|| die "HSTS_MAX_AGE_SECONDS должен быть не меньше 31536000"
[[ "$PUBLIC_DOCKER_PORTS" =~ ^[0-9]+(,[0-9]+)*$ ]] \
|| die "PUBLIC_DOCKER_PORTS должен иметь вид 80,443"
}
check_os() {
step "Проверка операционной системы"
[[ -r /etc/os-release ]] || die "Не найден /etc/os-release"
# shellcheck disable=SC1091
source /etc/os-release
[[ "${ID:-}" == "ubuntu" ]] || die "Поддерживается только Ubuntu"
local major="${VERSION_ID%%.*}"
((major >= 24)) || die "Требуется Ubuntu 24.04 или новее"
log "Обнаружена ${PRETTY_NAME}"
}
update_system() {
step "Обновление системы и установка пакетов"
export DEBIAN_FRONTEND=noninteractive
apt-get update
if [[ "$SKIP_APT_UPGRADE" != "true" ]]; then
apt-get dist-upgrade -y
fi
apt-get install -y \
ca-certificates \
curl \
dos2unix \
fail2ban \
git \
gnupg \
iptables \
jq \
logrotate \
netcat-openbsd \
openssl \
python3 \
python3-venv \
rsync \
unattended-upgrades \
ufw
apt-get autoremove -y
}
configure_time() {
step "Настройка времени"
timedatectl set-timezone "$TIMEZONE"
timedatectl set-ntp true
}
create_deploy_user() {
step "Пользователь развертывания"
if ! id "$DEPLOY_USER" >/dev/null 2>&1; then
useradd --create-home --shell /bin/bash "$DEPLOY_USER"
log "Создан пользователь ${DEPLOY_USER}"
else
log "Пользователь ${DEPLOY_USER} уже существует"
fi
install -d -m 700 -o "$DEPLOY_USER" -g "$DEPLOY_USER" \
"/home/${DEPLOY_USER}/.ssh"
local source_user="${SUDO_USER:-}"
local source_keys=""
local target_keys="/home/${DEPLOY_USER}/.ssh/authorized_keys"
if [[ -n "$source_user" && "$source_user" != "root" ]]; then
source_keys="/home/${source_user}/.ssh/authorized_keys"
elif [[ -s /root/.ssh/authorized_keys ]]; then
source_user="root"
source_keys="/root/.ssh/authorized_keys"
fi
if [[ "$COPY_SSH_KEYS" == "true" && ! -s "$target_keys" && -s "$source_keys" ]]; then
install -m 600 -o "$DEPLOY_USER" -g "$DEPLOY_USER" "$source_keys" "$target_keys"
log "SSH-ключи скопированы от ${source_user}"
fi
if [[ ! -s "$target_keys" ]]; then
log "ПРЕДУПРЕЖДЕНИЕ: у ${DEPLOY_USER} отсутствует authorized_keys"
fi
}
configure_account_passwords() {
step "Блокировка локальных паролей привилегированных учетных записей"
if [[ "$LOCK_ACCOUNT_PASSWORDS" != "true" ]]; then
log "LOCK_ACCOUNT_PASSWORDS=false: локальные пароли root и ${DEPLOY_USER} не изменены"
return
fi
[[ -s "/home/${DEPLOY_USER}/.ssh/authorized_keys" ]] \
|| die "Нельзя заблокировать пароль ${DEPLOY_USER}: authorized_keys пользователя пуст"
passwd --lock root
passwd --lock "$DEPLOY_USER"
log "Локальные пароли root и ${DEPLOY_USER} заблокированы; вход по SSH-ключам сохранен"
}
configure_layout() {
step "Каталоги HAN Chat"
install -d -m 755 -o "$DEPLOY_USER" -g "$DEPLOY_USER" "$DEPLOY_DIR"
install -d -m 700 -o "$DEPLOY_USER" -g "$DEPLOY_USER" \
"${DEPLOY_DIR}/secrets" \
"${DEPLOY_DIR}/secrets/pg" \
"${DEPLOY_DIR}/backups"
local env_file="${DEPLOY_DIR}/.env"
if [[ -f "$env_file" ]]; then
chown "$DEPLOY_USER:$DEPLOY_USER" "$env_file"
chmod 600 "$env_file"
fi
}
configure_swap() {
step "Настройка swap"
if ((SWAP_SIZE_GB == 0)); then
log "Создание swap отключено"
return
fi
if swapon --show=NAME --noheadings | grep -qx '/swapfile'; then
log "Swap уже подключен"
return
fi
if [[ ! -f /swapfile ]]; then
fallocate -l "${SWAP_SIZE_GB}G" /swapfile
chmod 600 /swapfile
mkswap /swapfile
fi
swapon /swapfile
grep -q '^/swapfile ' /etc/fstab \
|| printf '/swapfile none swap sw 0 0\n' >>/etc/fstab
printf 'vm.swappiness = 10\n' >/etc/sysctl.d/99-han-chat-swappiness.conf
sysctl --system >/dev/null
}
configure_sysctl() {
step "Настройка сетевого стека"
cat >/etc/sysctl.d/99-han-chat-hardening.conf <<'EOF'
net.ipv4.ip_forward = 1
net.ipv4.tcp_syncookies = 1
net.ipv4.conf.all.accept_redirects = 0
net.ipv4.conf.default.accept_redirects = 0
net.ipv4.conf.all.send_redirects = 0
net.ipv4.conf.default.send_redirects = 0
net.ipv4.conf.all.rp_filter = 1
net.ipv4.conf.default.rp_filter = 1
net.ipv4.icmp_echo_ignore_broadcasts = 1
net.ipv4.tcp_fin_timeout = 30
EOF
sysctl --system >/dev/null
}
install_docker() {
step "Установка Docker Engine и Compose"
if ! command -v docker >/dev/null 2>&1; then
install -m 0755 -d /etc/apt/keyrings
curl -fsSL https://download.docker.com/linux/ubuntu/gpg \
| gpg --dearmor --yes -o /etc/apt/keyrings/docker.gpg
chmod a+r /etc/apt/keyrings/docker.gpg
# shellcheck disable=SC1091
source /etc/os-release
printf '%s\n' \
"deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/ubuntu ${VERSION_CODENAME} stable" \
>/etc/apt/sources.list.d/docker.list
apt-get update
apt-get install -y docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin
fi
install -d -m 755 /etc/docker
cat >/etc/docker/daemon.json <<'EOF'
{
"live-restore": true,
"log-driver": "json-file",
"log-opts": {
"max-size": "50m",
"max-file": "5"
},
"userland-proxy": false
}
EOF
systemctl enable --now docker
systemctl restart docker
usermod -aG docker "$DEPLOY_USER"
docker compose version >/dev/null \
|| die "Docker Compose plugin не установлен"
log "$(docker --version)"
log "$(docker compose version)"
}
configure_ufw() {
step "Настройка UFW"
if [[ "$RESET_UFW" == "true" ]]; then
ufw --force reset
fi
ufw default deny incoming
ufw default allow outgoing
ufw allow "${SSH_PORT}/tcp" comment 'HAN Chat SSH'
ufw limit "${SSH_PORT}/tcp" comment 'HAN Chat SSH rate limit'
ufw allow 80/tcp comment 'HAN Chat HTTP'
ufw allow 443/tcp comment 'HAN Chat HTTPS'
ufw logging medium
ufw --force enable
}
configure_fail2ban() {
step "Настройка fail2ban для SSH"
cat >/etc/fail2ban/jail.d/han-chat.local <<EOF
[DEFAULT]
bantime = 2h
findtime = 10m
maxretry = 5
backend = systemd
banaction = ufw
[sshd]
enabled = true
port = ${SSH_PORT}
maxretry = 3
EOF
systemctl enable --now fail2ban
systemctl restart fail2ban
}
configure_unattended_upgrades() {
step "Автоматические обновления безопасности"
cat >/etc/apt/apt.conf.d/51han-chat-unattended <<'EOF'
Unattended-Upgrade::Remove-Unused-Dependencies "true";
Unattended-Upgrade::Automatic-Reboot "false";
EOF
dpkg-reconfigure -f noninteractive unattended-upgrades
systemctl enable --now unattended-upgrades
}
configure_docker_firewall() {
step "Фильтрация опубликованных Docker-портов"
cat >/etc/default/han-chat-docker-firewall <<EOF
EXTERNAL_IF=${EXTERNAL_IF}
PUBLIC_DOCKER_PORTS=${PUBLIC_DOCKER_PORTS}
EOF
cat >/usr/local/sbin/han-chat-docker-firewall <<'FIREWALL'
#!/usr/bin/env bash
set -Eeuo pipefail
# shellcheck disable=SC1091
source /etc/default/han-chat-docker-firewall
external_if="${EXTERNAL_IF:-}"
if [[ -z "$external_if" ]]; then
external_if="$(ip -4 route show default | awk '{print $5; exit}')"
fi
[[ -n "$external_if" ]] || {
echo "Не удалось определить внешний интерфейс; задайте EXTERNAL_IF" >&2
exit 1
}
iptables -N HAN-CHAT-DOCKER 2>/dev/null || true
iptables -F HAN-CHAT-DOCKER
iptables -A HAN-CHAT-DOCKER -m conntrack --ctstate RELATED,ESTABLISHED -j RETURN
iptables -A HAN-CHAT-DOCKER -i lo -j RETURN
IFS=',' read -ra ports <<<"$PUBLIC_DOCKER_PORTS"
for port in "${ports[@]}"; do
[[ "$port" =~ ^[0-9]+$ ]] || {
echo "Некорректный порт: $port" >&2
exit 1
}
iptables -A HAN-CHAT-DOCKER -i "$external_if" -p tcp --dport "$port" -j RETURN
done
# Блокируется только новый входящий трафик с внешнего интерфейса в Docker bridge.
# Исходящий и межконтейнерный трафик этой цепочкой не затрагивается.
iptables -A HAN-CHAT-DOCKER -i "$external_if" -o docker+ -j DROP
iptables -A HAN-CHAT-DOCKER -i "$external_if" -o br+ -j DROP
iptables -A HAN-CHAT-DOCKER -j RETURN
while iptables -C DOCKER-USER -j HAN-CHAT-DOCKER 2>/dev/null; do
iptables -D DOCKER-USER -j HAN-CHAT-DOCKER
done
iptables -I DOCKER-USER 1 -j HAN-CHAT-DOCKER
FIREWALL
chmod 750 /usr/local/sbin/han-chat-docker-firewall
cat >/etc/systemd/system/han-chat-docker-firewall.service <<'EOF'
[Unit]
Description=HAN Chat firewall for Docker published ports
After=docker.service network-online.target
Wants=docker.service network-online.target
[Service]
Type=oneshot
ExecStart=/usr/local/sbin/han-chat-docker-firewall
RemainAfterExit=yes
[Install]
WantedBy=multi-user.target
EOF
install -d -m 755 /etc/systemd/system/docker.service.d
cat >/etc/systemd/system/docker.service.d/han-chat-firewall.conf <<'EOF'
[Service]
ExecStartPost=-/usr/local/sbin/han-chat-docker-firewall
EOF
systemctl daemon-reload
systemctl enable --now han-chat-docker-firewall.service
}
configure_ssh() {
step "Настройка SSH"
[[ -s /root/.ssh/authorized_keys || -s "/home/${DEPLOY_USER}/.ssh/authorized_keys" ]] \
|| die "Нельзя отключить парольный SSH-вход: не найден ни один authorized_keys"
cat >/etc/ssh/sshd_config.d/00-han-chat.conf <<EOF
PasswordAuthentication no
KbdInteractiveAuthentication no
PubkeyAuthentication yes
X11Forwarding no
MaxAuthTries 3
ClientAliveInterval 120
ClientAliveCountMax 2
Port ${SSH_PORT}
EOF
if [[ "$HARDEN_SSH" == "true" ]]; then
cat >>/etc/ssh/sshd_config.d/00-han-chat.conf <<'EOF'
PermitRootLogin no
AllowTcpForwarding no
EOF
log "Расширенный SSH hardening включен: root-вход и TCP forwarding запрещены"
else
log "Базовый SSH hardening включен; root-вход и TCP forwarding не изменены"
fi
rm -f /etc/ssh/sshd_config.d/99-han-chat.conf
sshd -t || die "Проверка конфигурации sshd не пройдена"
systemctl reload ssh
}
configure_application_security() {
step "Безопасные HTTP-заголовки приложения"
local env_file="${DEPLOY_DIR}/.env"
if [[ -f "$env_file" ]]; then
if grep -q '^NGINX_HSTS_MAX_AGE=' "$env_file"; then
sed -i "s/^NGINX_HSTS_MAX_AGE=.*/NGINX_HSTS_MAX_AGE=${HSTS_MAX_AGE_SECONDS}/" "$env_file"
else
printf '\nNGINX_HSTS_MAX_AGE=%s\n' "$HSTS_MAX_AGE_SECONDS" >>"$env_file"
fi
chown "$DEPLOY_USER:$DEPLOY_USER" "$env_file"
chmod 600 "$env_file"
log "HSTS настроен на ${HSTS_MAX_AGE_SECONDS} секунд в ${env_file}"
else
log "Проект еще не настроен: HSTS будет взят из безопасного значения Compose по умолчанию"
fi
}
install_secret_loader_if_possible() {
step "Загрузчик секретов"
local source_dir="${DEPLOY_DIR}/deployment/secrets"
if [[ ! -f "${source_dir}/secrets_loader.py" || ! -f "${source_dir}/han-secrets" ]]; then
log "Проект еще не скопирован: загрузчик секретов будет установлен при повторном запуске"
return
fi
chmod 0750 "${source_dir}/han-secrets" "${source_dir}/han-compose"
install -d -m 0700 -o root -g root \
/etc/han \
/etc/han/secrets \
/etc/han/credentials
install -d -m 0755 -o root -g root \
/usr/local/lib/han-secrets \
/usr/local/share/doc/han-secrets
install -m 0750 -o root -g root \
"${source_dir}/secrets_loader.py" \
/usr/local/lib/han-secrets/secrets_loader.py
install -m 0750 -o root -g root \
"${source_dir}/han-secrets" \
/usr/local/lib/han-secrets/han-secrets
install -m 0750 -o root -g root \
"${source_dir}/han-compose" \
/usr/local/bin/han-compose
install -m 0644 -o root -g root \
"${source_dir}/han-secrets@.service" \
/etc/systemd/system/han-secrets@.service
install -m 0644 -o root -g root \
"${source_dir}/SELECTEL_RUNBOOK.ru.md" \
/usr/local/share/doc/han-secrets/SELECTEL_RUNBOOK.ru.md
if [[ ! -e /etc/han/secrets/production.selectel.json.example ]]; then
install -m 0600 -o root -g root \
"${source_dir}/config.example.json" \
/etc/han/secrets/production.selectel.json.example
fi
systemctl daemon-reload
log "Загрузчик установлен, но не включен: сначала выполните SELECTEL_RUNBOOK.ru.md"
}
install_ssl_timer_if_possible() {
step "Таймер продления TLS"
local renew_script="${DEPLOY_DIR}/deployment/scripts/ssl-renew.sh"
if [[ ! -x "$renew_script" ]]; then
log "Проект еще не скопирован: таймер TLS будет установлен при повторном запуске"
return
fi
cat >/etc/systemd/system/han-chat-ssl-renew.service <<EOF
[Unit]
Description=Renew HAN Chat TLS certificate
After=docker.service
[Service]
Type=oneshot
User=${DEPLOY_USER}
WorkingDirectory=${DEPLOY_DIR}
ExecStart=${renew_script}
EOF
cat >/etc/systemd/system/han-chat-ssl-renew.timer <<'EOF'
[Unit]
Description=Run HAN Chat TLS renewal twice daily
[Timer]
OnCalendar=*-*-* 03,15:20:00
RandomizedDelaySec=30m
Persistent=true
[Install]
WantedBy=timers.target
EOF
systemctl daemon-reload
systemctl enable --now han-chat-ssl-renew.timer
}
verify() {
step "Проверка результата"
local failed=0
systemctl is-active --quiet docker || { log "FAIL: Docker не активен"; failed=1; }
systemctl is-active --quiet fail2ban || { log "FAIL: fail2ban не активен"; failed=1; }
ufw status | grep -q 'Status: active' || { log "FAIL: UFW не активен"; failed=1; }
iptables -C DOCKER-USER -j HAN-CHAT-DOCKER 2>/dev/null \
|| { log "FAIL: цепочка HAN-CHAT-DOCKER не подключена"; failed=1; }
docker compose version >/dev/null || { log "FAIL: Compose недоступен"; failed=1; }
[[ -d "$DEPLOY_DIR" ]] || { log "FAIL: отсутствует ${DEPLOY_DIR}"; failed=1; }
((failed == 0)) || die "Базовая проверка VM не пройдена"
log "Базовая проверка VM пройдена"
}
summary() {
step "Настройка VM завершена"
cat <<EOF | tee -a "$LOG_FILE"
Пользователь развертывания: ${DEPLOY_USER}
Каталог Compose: ${DEPLOY_DIR}
Открытые порты: ${SSH_PORT}, 80, 443
Парольный SSH/X11: отключены
Локальные пароли: ${LOCK_ACCOUNT_PASSWORDS}
HSTS max-age: ${HSTS_MAX_AGE_SECONDS}
Лог настройки: ${LOG_FILE}
Следующие действия:
1. Проверьте вход в новой SSH-сессии:
ssh ${DEPLOY_USER}@<VM_IP>
2. Скопируйте содержимое codebase/backend в:
${DEPLOY_DIR}
3. Поместите CA PostgreSQL:
${DEPLOY_DIR}/secrets/pg/ca.pem
4. Создайте только несекретный config:
cd ${DEPLOY_DIR}
cp .env.example .env
chmod 600 .env
./scripts/validate-env .env
5. Настройте Selectel, encrypted bootstrap credential и fallback map:
deployment/secrets/SELECTEL_RUNBOOK.ru.md
6. Выполняйте Compose только через:
sudo deployment/secrets/han-compose <command>
7. Продолжите с Gate 7 в:
deployment/RUNBOOK.ru.md
8. После копирования проекта повторно запустите этот скрипт для установки unit-файлов.
Важно: членство в группе docker начнет действовать после нового входа в систему.
EOF
}
main() {
require_root
install -d -m 755 "$(dirname "$LOG_FILE")"
touch "$LOG_FILE"
chmod 600 "$LOG_FILE"
validate_parameters
check_os
update_system
configure_time
create_deploy_user
configure_account_passwords
configure_layout
configure_swap
configure_sysctl
install_docker
configure_ufw
configure_fail2ban
configure_unattended_upgrades
configure_docker_firewall
configure_ssh
configure_application_security
install_secret_loader_if_possible
install_ssl_timer_if_possible
verify
summary
}
main "$@"
VM1_app/codebase/backend/deployment/scripts/smoke.sh
+71
View File
@@ -0,0 +1,71 @@
#!/bin/sh
set -eu
cd "$(dirname "$0")/../.."
CONFIG_FILE=${CONFIG_FILE:-.env}
SECRETS_LAUNCHER=${SECRETS_LAUNCHER:-deployment/secrets/han-secrets}
if [ "${HAN_SECRETS_ACTIVE:-0}" != "1" ]; then
[ -x "$SECRETS_LAUNCHER" ] || {
echo "Secret launcher is required: $SECRETS_LAUNCHER" >&2
exit 66
}
exec "$SECRETS_LAUNCHER" run --config "$CONFIG_FILE" -- "$0" "$@"
fi
if [ -n "${HAN_RUNTIME_SECRET_MANIFEST:-}" ]; then
./scripts/validate-env "$CONFIG_FILE" --runtime-manifest "$HAN_RUNTIME_SECRET_MANIFEST"
else
./scripts/validate-env "$CONFIG_FILE" --runtime-env
fi
env_value() {
python3 - "$CONFIG_FILE" "$1" <<'PY'
import sys
from pathlib import Path
path, wanted = sys.argv[1:]
for raw in Path(path).read_text(encoding="utf-8").splitlines():
line = raw.strip()
if not line or line.startswith("#") or "=" not in line:
continue
key, value = line.split("=", 1)
if key.strip() == wanted:
value = value.strip()
if len(value) >= 2 and value[0] == value[-1] and value[0] in "\"'":
value = value[1:-1]
print(value)
break
else:
raise SystemExit(f"missing environment variable: {wanted}")
PY
}
PUBLIC_HOST=$(env_value PUBLIC_HOST)
PUBLIC_WEB_URL=$(env_value PUBLIC_WEB_URL)
KEYCLOAK_REALM=$(env_value KEYCLOAK_REALM)
tmp=$(mktemp -d)
trap 'rm -rf "$tmp"' EXIT
http_code=$(curl -sS -o /dev/null -w '%{http_code}' "http://${PUBLIC_HOST}/")
[ "$http_code" = "308" ] || { echo "Expected HTTP 308, got $http_code" >&2; exit 1; }
curl -fsS "${PUBLIC_WEB_URL}/api/v1/public/app-config" -o "$tmp/app-config.json"
curl -fsS "${PUBLIC_WEB_URL}/api/v1/public/content" -o "$tmp/content.json"
curl -fsS "${PUBLIC_WEB_URL}/auth/realms/${KEYCLOAK_REALM}/.well-known/openid-configuration" -o "$tmp/oidc.json"
internal_code=$(curl -sS -o /dev/null -w '%{http_code}' "${PUBLIC_WEB_URL}/internal/safety/v1/messages/check")
[ "$internal_code" = "404" ] || { echo "Public /internal returned $internal_code, expected 404" >&2; exit 1; }
sms_internal_code=$(curl -sS -o /dev/null -w '%{http_code}' "${PUBLIC_WEB_URL}/internal/sms/v1/messages/00000000-0000-0000-0000-000000000000")
[ "$sms_internal_code" = "404" ] || { echo "Public SMS internal API returned $sms_internal_code, expected 404" >&2; exit 1; }
sms_callback_code=$(curl -sS -o /dev/null -w '%{http_code}' -X POST \
-H 'Content-Type: application/json' --data '[]' \
"${PUBLIC_WEB_URL}/callbacks/idgtl/sms")
[ "$sms_callback_code" = "403" ] || { echo "SMS callback without provider IP returned $sms_callback_code, expected 403" >&2; exit 1; }
headers=$(curl -fsSI "${PUBLIC_WEB_URL}/")
printf '%s' "$headers" | grep -qi '^x-content-type-options: nosniff'
printf '%s' "$headers" | grep -qi '^x-request-id:'
printf '%s' "$headers" | grep -qi '^content-security-policy:'
openssl s_client -connect "${PUBLIC_HOST}:443" -servername "$PUBLIC_HOST" </dev/null 2>/dev/null \
| openssl x509 -noout -checkend 604800
echo "Public edge smoke passed."
VM1_app/codebase/backend/deployment/scripts/ssl-renew.sh
+38
View File
@@ -0,0 +1,38 @@
#!/bin/sh
set -eu
cd "$(dirname "$0")/../.."
CONFIG_FILE=${CONFIG_FILE:-.env}
SECRETS_LAUNCHER=${SECRETS_LAUNCHER:-deployment/secrets/han-secrets}
if [ "${HAN_SECRETS_ACTIVE:-0}" != "1" ]; then
[ -x "$SECRETS_LAUNCHER" ] || {
echo "Secret launcher is required: $SECRETS_LAUNCHER" >&2
exit 66
}
exec "$SECRETS_LAUNCHER" run --config "$CONFIG_FILE" -- "$0" "$@"
fi
lock=/tmp/han-chat-cert-renew.lock
exec 9>"$lock"
flock -n 9 || { echo '{"event":"tls.renew.skipped","reason":"lock_busy"}'; exit 0; }
compose() {
docker compose --env-file "$CONFIG_FILE" "$@"
}
nginx_container="$(compose ps --status running --quiet nginx)"
if [ -z "$nginx_container" ]; then
echo '{"event":"tls.renew.failed","reason":"nginx_not_running"}' >&2
exit 1
fi
compose --profile certbot run --rm certbot renew \
--webroot -w /var/www/certbot --quiet
compose exec -T nginx nginx -t -c /tmp/nginx.conf
# Сигнал отправляется PID 1 контейнера. Нельзя использовать `nginx -s reload`:
# он ищет дефолтный /var/run/nginx.pid, тогда как рабочий PID — /tmp/nginx.pid.
compose kill --signal HUP nginx
compose ps --status running --quiet nginx | awk 'NF {found=1} END {exit !found}'
echo "{\"event\":\"tls.renew.completed\",\"timestamp\":\"$(date -u +%FT%TZ)\"}"
VM1_app/codebase/backend/deployment/scripts/verify-observability.sh
+75
View File
@@ -0,0 +1,75 @@
#!/usr/bin/env bash
set -Eeuo pipefail
cd "$(dirname "$0")/../.."
CONFIG_FILE="${CONFIG_FILE:-.env}"
SECRETS_LAUNCHER="${SECRETS_LAUNCHER:-deployment/secrets/han-secrets}"
if [[ "${HAN_SECRETS_ACTIVE:-0}" != 1 ]]; then
[[ -x "$SECRETS_LAUNCHER" ]] || {
echo "Secret launcher is required: $SECRETS_LAUNCHER" >&2
exit 66
}
exec "$SECRETS_LAUNCHER" run --config "$CONFIG_FILE" -- "$0" "$@"
fi
compose() { docker compose --env-file "$CONFIG_FILE" "$@"; }
NETWORK="${OBSERVABILITY_NETWORK:-han-chat-observability}"
COLLECTOR_SERVICE="${COLLECTOR_SERVICE:-otel-collector}"
errors=0
ok() { printf 'OK %s\n' "$*"; }
fail() { printf 'FAIL %s\n' "$*" >&2; errors=$((errors + 1)); }
collector_id="$(compose ps -q "$COLLECTOR_SERVICE" 2>/dev/null || true)"
if [[ -n "$collector_id" ]] &&
[[ "$(docker inspect --format '{{.State.Status}}' "$collector_id")" == running ]]; then
ok "Collector service is running"
else
fail "Collector service '$COLLECTOR_SERVICE' is not running"
fi
for service in sms-service sms-worker; do
if compose exec -T "$service" python - <<'PY' >/dev/null 2>&1
import os
import urllib.request
port = os.environ.get("SMS_METRICS_PORT", "9464") if "worker" in os.environ.get("OTEL_SERVICE_NAME", "") else "8080"
urllib.request.urlopen(f"http://127.0.0.1:{port}/metrics", timeout=3).read(1024)
PY
then
ok "${service} metrics endpoint"
else
fail "${service} metrics endpoint"
fi
done
docker run --rm --network "$NETWORK" \
ghcr.io/open-telemetry/opentelemetry-collector-contrib/telemetrygen:latest \
traces --otlp-endpoint otel-collector:4317 --otlp-insecure \
--service han-chat-e2e-canary --traces 100 --rate 20 >/dev/null \
&& ok "100 canary traces submitted" \
|| fail "telemetrygen failed"
bad_logs="$(
compose logs --since=10m "$COLLECTOR_SERVICE" 2>&1 |
grep -Ei 'queue is full|connection refused|tls:|Unauthenticated|Permanent error' || true
)"
if [[ -z "$bad_logs" ]]; then
ok "No exporter/queue errors in last 10 minutes"
else
fail "Collector reports exporter/queue errors"
printf '%s\n' "$bad_logs" >&2
fi
if ((errors)); then
printf 'Observability verification failed: %d check(s)\n' "$errors" >&2
exit 1
fi
cat <<'EOF'
Локальный канал исправен. В SigNoz проверьте за последние 15 минут:
service.name = han-chat-e2e-canary
service.namespace = han-chat
Затем выполните synthetic API request и проверьте общий trace между
api-backend и dependency spans, service.version и deployment.environment.
EOF
VM1_app/codebase/backend/deployment/secrets/SELECTEL_RUNBOOK.ru.md
+206
View File
@@ -0,0 +1,206 @@
# Selectel Secrets Manager для HAN Chat
## Модель
`han-secrets` читает только `SECRETS_SOURCE=selectel|file` из обычного `.env`,
выбирает соответствующую root-only JSON-карту и вызывает `secrets_loader.py`.
Загрузчик получает project-scoped IAM token, читает объявленные секреты и
создаёт в `/run/han-chat/secrets`:
- отдельные файлы с каноническими именами для Compose secrets;
- узкие service dotenv-файлы для диагностики состава без вывода значений;
- `manifest` вида `NAME=/absolute/path`, используемый валидатором.
Каталог `/run` находится в tmpfs и имеет режим `0700`. Канонические файлы имеют
`0444`: локальные пользователи не могут пройти через root-only каталог, а
не-root UID контейнера может прочитать только явно смонтированный Compose
secret. Значения не передаются через Docker Config.Env, argv, общий `.env` или
логи. После полного root/docker-компромисса runtime-значения извлекаемы — это
ограничение модели, а не гарантия Secret Manager.
Сбой Selectel никогда автоматически не включает file fallback. Уже работающие
контейнеры продолжают использовать текущие значения; новый sync завершается
fail-closed.
## 1. Ресурсы Selectel
1. Создайте отдельный проект `han-chat-secrets-prod`.
2. Создайте сервисного пользователя `han-chat-secrets-reader`.
3. Назначьте ему `member` только в этом проекте. Не выдавайте account scope,
`iam.admin` и доступ к другим production-ресурсам. Если Selectel добавит
отдельную read-only роль Secrets Manager, замените `member` на неё.
4. Ограничьте обращения к `api.selectel.ru` исходящим IP ВМ, если функция
доступна в аккаунте.
5. Включите экспорт audit logs. Контролируйте события `secrets.secret*` и
`secrets.secret_version*`; alert на delete, смену current version вне окна,
массовые чтения и обращения не от штатного пользователя/IP.
6. Выполните canary и убедитесь, что provider audit содержит metadata операции,
но не value, Base64 payload, IAM token или response body.
Secrets Manager принимает project-scoped IAM token в `X-Auth-Token`. Token
живёт до 24 часов, но загрузчик использует его только в памяти одного запуска.
TLS и redirect policy отключать нельзя.
## 2. Каталог секретов
Скопируйте `config.example.json` в
`/etc/han/secrets/production-like.selectel.json` и замените account, username,
project, region и `remote`. Каноническое имя слева обязано совпадать с
Compose/validator; `remote` — неизменяемый ключ в Selectel.
Используйте консервативные provider keys с дефисами, например:
- `han-chat-prod-pg-han-app-dsn`, `han-chat-prod-pg-bitrix-local-dsn`,
`han-chat-prod-pg-bitrix-sync-dsn`, `han-chat-prod-pg-sms-dsn`,
`han-chat-prod-pg-keycloak-password`, `han-chat-prod-pg-backup-dsn`;
- `han-chat-prod-redis-api-password`, `han-chat-prod-redis-safety-password`,
`han-chat-prod-redis-health-password`, а также три credential-bearing URL;
- `han-chat-prod-message-safety-token`, `han-chat-prod-bitrix-internal-token`,
`han-chat-prod-bitrix-forward-token`, `han-chat-prod-bitrix-sync-token`,
`han-chat-prod-keycloak-settings-token`, `han-chat-prod-sms-service-token`;
- Keycloak bootstrap password, OTP HMAC и mock code только для среды, где mock
действительно включён;
- Bitrix client secret, application token и token encryption key;
- пары S3 app access/secret key;
- i-Digital API key и callback username/password.
Одинаковые пары env (`BITRIX_LOCAL_APP_INTERNAL_TOKEN` /
`BITRIX_INTERNAL_API_TOKEN`, `BITRIX_API_FORWARD_TOKEN` /
`BITRIX_API_INBOX_TOKEN`, `SMS_SERVICE_TOKEN` /
`KEYCLOAK_SMS_SERVICE_TOKEN`) должны ссылаться на один `remote`.
`literal: ""` разрешён только для заведомо пустого optional-параметра, например
OTLP auth при self-hosted SigNoz или выключенного CAPTCHA server key. Секреты
не записывайте в JSON. Не заводите planned/unused Message Safety DB, quarantine
S3 и Bitrix sync credentials до появления потребляющего кода.
Загружайте значения в Selectel через скрытый prompt/stdin. Не передавайте value
позиционным аргументом CLI и не включайте `set -x`/`curl -v`.
## 3. Установка на ВМ
Повторный запуск `deployment/scripts/setup-vm.sh` после копирования проекта
устанавливает loader, launcher, systemd template и этот runbook. Вручную:
```sh
sudo install -d -m 0700 /etc/han/secrets /etc/han/credentials
sudo install -d -m 0755 /usr/local/lib/han-secrets
sudo install -m 0750 secrets_loader.py han-secrets /usr/local/lib/han-secrets/
sudo install -m 0750 han-compose /usr/local/bin/han-compose
sudo install -m 0644 han-secrets@.service /etc/systemd/system/
sudo install -m 0600 config.example.json \
/etc/han/secrets/production-like.selectel.json
```
Обычный `/opt/han-chat/backend/.env` содержит только несекретные параметры.
Штатный режим:
```dotenv
SECRETS_SOURCE=selectel
APP_ENV=production-like
```
## 4. Bootstrap credential
Не храните пароль service user в `.env` или JSON. На целевой ВМ:
```sh
sudo systemd-creds encrypt --name=selectel-service-user-password - \
/etc/han/credentials/production.selectel-password.cred
sudo chmod 0600 /etc/han/credentials/production.selectel-password.cred
```
Введите пароль через интерактивный stdin. Unit передаёт расшифрованный файл
через приватный `$CREDENTIALS_DIRECTORY`. Root всё равно может его извлечь;
после инцидента credential и все доступные ему секреты необходимо ротировать.
## 5. Проверка и запуск
Все команды, которым нужны Compose secrets, запускайте от root через wrapper:
```sh
cd /opt/han-chat/backend
sudo ./scripts/validate-env .env
sudo systemctl daemon-reload
sudo systemctl enable han-secrets@production.service
sudo systemctl restart han-secrets@production.service
sudo ./scripts/validate-env .env \
--runtime-manifest /run/han-chat/secrets/manifest
sudo deployment/secrets/han-compose config --quiet
sudo deployment/secrets/han-compose up -d --wait
```
Selectel sync запускается именно unit-файлом: только он предоставляет
расшифрованный bootstrap credential через `$CREDENTIALS_DIRECTORY`.
`han-compose` и ops-скрипты используют уже синхронизированный manifest и
отказываются работать, если `SECRETS_SOURCE`/loader config не совпадают с
runtime state. После смены source, provider version или JSON-карты сначала
выполняйте `systemctl restart han-secrets@production.service`.
Не используйте `docker compose config` без `--quiet`, `docker inspect` для
поиска конфигурации, `env`, `strace`, core dump или debug HTTP proxy. Проверка
приёмки должна подтвердить отсутствие canary value в `docker inspect`, stdout,
json logs, traces и shell history.
На выделенной только под HAN Chat ВМ после canary и проверки file fallback
установите fail-closed ordering:
```sh
sudo install -d -m 0755 /etc/systemd/system/docker.service.d
sudo install -m 0644 \
deployment/secrets/docker-han-secrets.conf.example \
/etc/systemd/system/docker.service.d/han-secrets.conf
sudo systemctl daemon-reload
sudo systemctl restart docker
```
После этого проведите reboot rehearsal: materializer должен завершиться до
autorestart контейнеров. Ошибка Selectel намеренно блокирует Docker. На ВМ с
другими workloads такой глобальный `Requires=` запрещён: нужен отдельный Docker
daemon/VM, иначе fail-closed HAN остановит несвязанные системы.
## 6. Явный file fallback
Подготовьте отдельную карту
`/etc/han/secrets/production-like.file.json`: скопируйте Selectel-карту,
установите `"mode": "file"`, удалите `selectel` и `http`, добавьте:
```json
"file": {
"path": "/etc/han/break-glass/secrets.env",
"max_bytes": 1048576
}
```
`secrets` map остаётся тем же. Для записей с `literal: ""` строка в fallback
не нужна; остальные канонические ключи обязательны. Fallback parser не исполняет
shell: запрещены `export`, substitutions, multiline, неизвестные и дублирующиеся
ключи. Файл — `root:root 0600`.
При инциденте доставьте recovery-файл из защищённой офлайн-копии и только затем
явно измените `.env`:
```dotenv
SECRETS_SOURCE=file
```
Перезапустите `han-secrets@production.service`, затем выполните
validate/recreate через wrappers. После восстановления Selectel верните
`SECRETS_SOURCE=selectel`, снова перезапустите unit, повторите проверки и
удалите recovery-файл.
Не храните его постоянно на ВМ: это вернуло бы исходный риск монолитного `.env`.
## 7. Ротация и rollback
1. Добавьте новую версию секрета, не меняя имя.
2. Для canary при необходимости временно pin числовой `version` в JSON-карте.
3. Выполните sync, validation и smoke без вывода конфигурации.
4. Сделайте версию current, удалите pin, снова sync и пересоздайте только
потребителей.
5. Для rollback активируйте предыдущую provider version; не храните snapshot
старого `.env`.
Selectel не позволяет удалить отдельную версию — только секрет целиком. Старые
значения должны быть отозваны в PostgreSQL/S3/Bitrix/i-Digital после окна
rollback. Все значения из прежнего `.env` считайте раскрытыми и ротируйте после
перехода.
VM1_app/codebase/backend/deployment/secrets/config.example.json
+190
View File
@@ -0,0 +1,190 @@
{
"version": 1,
"mode": "selectel",
"runtime_dir": "/run/han-chat/secrets",
"http": {
"timeout_seconds": 10,
"retries": 3,
"max_response_bytes": 1048576
},
"selectel": {
"account_id": "123456",
"username": "han-secrets-reader",
"project_name": "han-production",
"region": "ru-9",
"interface": "public",
"password_file": "selectel-service-user-password"
},
"secrets": {
"DATABASE_URL": {
"remote": "DATABASE_URL",
"consumers": ["api-backend", "api-migrate"],
"max_bytes": 4096
},
"BITRIX_DATABASE_URL": {
"remote": "BITRIX_DATABASE_URL",
"consumers": ["bitrix-local-app", "bitrix-local-migrate"],
"max_bytes": 4096
},
"BITRIX_SYNC_DATABASE_URL": {
"remote": "BITRIX_SYNC_DATABASE_URL",
"consumers": ["bitrix-sync", "bitrix-sync-migrate"],
"max_bytes": 4096
},
"SMS_DATABASE_URL": {
"remote": "SMS_DATABASE_URL",
"consumers": ["sms-service", "sms-worker", "sms-migrate"],
"max_bytes": 4096
},
"KEYCLOAK_DB_PASSWORD": {
"remote": "KEYCLOAK_DB_PASSWORD",
"consumers": ["keycloak"],
"max_bytes": 1024
},
"KEYCLOAK_ADMIN_PASSWORD": {
"remote": "KEYCLOAK_ADMIN_PASSWORD",
"consumers": ["keycloak"],
"max_bytes": 1024
},
"CURSOR_HMAC_SECRET": {
"remote": "CURSOR_HMAC_SECRET",
"consumers": ["api-backend"],
"max_bytes": 1024
},
"KEYCLOAK_OTP_HMAC_KEY": {
"remote": "KEYCLOAK_OTP_HMAC_KEY",
"consumers": ["keycloak"],
"max_bytes": 1024
},
"KEYCLOAK_OTP_MOCK_CODE": {
"remote": "KEYCLOAK_OTP_MOCK_CODE",
"consumers": ["keycloak"],
"max_bytes": 1024
},
"KEYCLOAK_YANDEX_CAPTCHA_SERVER_KEY": {
"remote": "KEYCLOAK_YANDEX_CAPTCHA_SERVER_KEY",
"consumers": ["keycloak"],
"max_bytes": 1024
},
"BITRIX_TOKEN_ENCRYPTION_KEY": {
"remote": "BITRIX_TOKEN_ENCRYPTION_KEY",
"consumers": ["api-backend", "bitrix-local-app", "bitrix-sync"],
"max_bytes": 1024
},
"REDIS_API_PASSWORD": {
"remote": "REDIS_API_PASSWORD",
"consumers": ["redis"],
"max_bytes": 1024
},
"REDIS_URL": {
"remote": "REDIS_URL",
"consumers": ["api-backend", "delivery-worker", "cleanup-worker"],
"max_bytes": 4096
},
"REDIS_REALTIME_URL": {
"remote": "REDIS_REALTIME_URL",
"consumers": ["api-backend"],
"max_bytes": 4096
},
"REDIS_SAFETY_PASSWORD": {
"remote": "REDIS_SAFETY_PASSWORD",
"consumers": ["redis"],
"max_bytes": 1024
},
"MESSAGE_SAFETY_REDIS_URL": {
"remote": "MESSAGE_SAFETY_REDIS_URL",
"consumers": ["message-safety", "safety-recovery-worker"],
"max_bytes": 4096
},
"REDIS_HEALTH_PASSWORD": {
"remote": "REDIS_HEALTH_PASSWORD",
"consumers": ["redis", "redis-exporter"],
"max_bytes": 1024
},
"MESSAGE_SAFETY_SERVICE_TOKEN": {
"remote": "MESSAGE_SAFETY_SERVICE_TOKEN",
"consumers": ["api-backend", "message-safety"],
"max_bytes": 1024
},
"BITRIX_LOCAL_APP_INTERNAL_TOKEN": {
"remote": "BITRIX_LOCAL_APP_INTERNAL_TOKEN",
"consumers": ["api-backend", "bitrix-local-app"],
"max_bytes": 1024
},
"BITRIX_INTERNAL_API_TOKEN": {
"remote": "BITRIX_LOCAL_APP_INTERNAL_TOKEN",
"consumers": ["api-backend", "bitrix-local-app"],
"max_bytes": 1024
},
"BITRIX_API_FORWARD_TOKEN": {
"remote": "BITRIX_API_FORWARD_TOKEN",
"consumers": ["api-backend", "bitrix-local-app"],
"max_bytes": 1024
},
"BITRIX_API_INBOX_TOKEN": {
"remote": "BITRIX_API_FORWARD_TOKEN",
"consumers": ["api-backend", "bitrix-local-app"],
"max_bytes": 1024
},
"BITRIX_SYNC_SERVICE_TOKEN": {
"remote": "BITRIX_SYNC_SERVICE_TOKEN",
"consumers": ["api-backend", "bitrix-sync"],
"max_bytes": 1024
},
"KEYCLOAK_SETTINGS_BRIDGE_TOKEN": {
"remote": "KEYCLOAK_SETTINGS_BRIDGE_TOKEN",
"consumers": ["api-backend", "keycloak"],
"max_bytes": 1024
},
"SMS_SERVICE_TOKEN": {
"remote": "SMS_SERVICE_TOKEN",
"consumers": ["sms-service", "sms-worker", "keycloak"],
"max_bytes": 1024
},
"KEYCLOAK_SMS_SERVICE_TOKEN": {
"remote": "SMS_SERVICE_TOKEN",
"consumers": ["sms-service", "sms-worker", "keycloak"],
"max_bytes": 1024
},
"IDGTL_SMS_API_KEY": {
"remote": "IDGTL_SMS_API_KEY",
"consumers": ["sms-service", "sms-worker"],
"max_bytes": 1024
},
"IDGTL_SMS_CALLBACK_USERNAME": {
"remote": "IDGTL_SMS_CALLBACK_USERNAME",
"consumers": ["sms-service", "sms-worker"],
"max_bytes": 1024
},
"IDGTL_SMS_CALLBACK_PASSWORD": {
"remote": "IDGTL_SMS_CALLBACK_PASSWORD",
"consumers": ["sms-service", "sms-worker"],
"max_bytes": 1024
},
"BITRIX_CLIENT_SECRET": {
"remote": "BITRIX_CLIENT_SECRET",
"consumers": ["bitrix-local-app"],
"max_bytes": 1024
},
"BITRIX_APPLICATION_TOKEN": {
"remote": "BITRIX_APPLICATION_TOKEN",
"consumers": ["bitrix-local-app"],
"max_bytes": 1024
},
"SELECTEL_S3_SECRET_KEY": {
"remote": "SELECTEL_S3_SECRET_KEY",
"consumers": ["api-backend"],
"max_bytes": 1024
},
"SELECTEL_S3_ACCESS_KEY": {
"remote": "SELECTEL_S3_ACCESS_KEY",
"consumers": ["api-backend"],
"max_bytes": 1024
},
"OTEL_REMOTE_AUTH_HEADER": {
"literal": "",
"consumers": ["otel-collector"],
"max_bytes": 4096
}
}
}
VM1_app/codebase/backend/deployment/secrets/docker-han-secrets.conf.example
+6
View File
@@ -0,0 +1,6 @@
[Unit]
# Enable only on a dedicated HAN Chat Docker host after the Selectel canary and
# file fallback have both passed. A failed secret sync intentionally blocks
# Docker startup so containers cannot race an empty /run directory.
Requires=han-secrets@production.service
After=han-secrets@production.service
VM1_app/codebase/backend/deployment/secrets/han-compose
+15
View File
@@ -0,0 +1,15 @@
#!/bin/sh
set -eu
DEPLOY_DIR=${HAN_DEPLOY_DIR:-/opt/han-chat/backend}
SOURCE_DIR=$(CDPATH= cd -- "$(dirname "$0")" && pwd)
[ ! -f "$SOURCE_DIR/../../docker-compose.yml" ] || \
DEPLOY_DIR=$(CDPATH= cd -- "$SOURCE_DIR/../.." && pwd)
cd "$DEPLOY_DIR"
CONFIG_FILE=${CONFIG_FILE:-.env}
LAUNCHER=${HAN_SECRETS_LAUNCHER:-/usr/local/lib/han-secrets/han-secrets}
[ -x "$LAUNCHER" ] || LAUNCHER=deployment/secrets/han-secrets
exec "$LAUNCHER" run --config "$CONFIG_FILE" -- \
docker compose --env-file "$CONFIG_FILE" "$@"
VM1_app/codebase/backend/deployment/secrets/han-secrets
+175
View File
@@ -0,0 +1,175 @@
#!/usr/bin/env python3
"""Synchronize runtime secrets and execute a command without exporting their values."""
from __future__ import annotations
import argparse
import json
import os
import subprocess
import sys
import tempfile
from pathlib import Path
from secrets_loader import LoaderError, load_json, run
def load_public_config(path: Path) -> dict[str, str]:
values: dict[str, str] = {}
try:
lines = path.read_text(encoding="utf-8").splitlines()
except OSError as exc:
raise LoaderError(
f"cannot read non-secret config: {exc.strerror or exc.__class__.__name__}"
) from None
for number, raw in enumerate(lines, 1):
line = raw.strip()
if not line or line.startswith("#"):
continue
if "=" not in line:
raise LoaderError(f"non-secret config has invalid syntax at line {number}")
key, value = line.split("=", 1)
key = key.strip()
if not key or key in values:
raise LoaderError(f"non-secret config has an invalid key at line {number}")
values[key] = value.strip()
return values
def loader_config_path(
public: dict[str, str],
explicit: Path | None,
environ: dict[str, str],
) -> tuple[str, Path]:
source = public.get("SECRETS_SOURCE")
if source not in {"selectel", "file"}:
raise LoaderError("SECRETS_SOURCE must explicitly be selectel or file")
if explicit is not None:
return source, explicit
override = environ.get(f"HAN_SECRETS_{source.upper()}_CONFIG")
if override:
return source, Path(override)
environment = public.get("APP_ENV", "production")
return source, Path(f"/etc/han/secrets/{environment}.{source}.json")
def prepare_environment(
config_path: Path,
source: str,
environ: dict[str, str],
*,
synchronize: bool,
) -> dict[str, str]:
document = load_json(config_path)
if document.get("mode") != source:
raise LoaderError("selected loader configuration mode does not match SECRETS_SOURCE")
runtime = document.get("runtime_dir")
if not isinstance(runtime, str) or not Path(runtime).is_absolute():
raise LoaderError("loader configuration has an invalid runtime_dir")
runtime_dir = Path(runtime)
manifest = runtime_dir / "manifest"
state_path = runtime_dir / "state.json"
consumers: list[str] = []
if synchronize or (source == "file" and not state_path.is_file()):
consumers = run(config_path, environ=environ)
state = {
"version": 1,
"source": source,
"loader_config": str(config_path.resolve()),
}
descriptor, temporary = tempfile.mkstemp(
prefix=".state.", suffix=".tmp", dir=runtime_dir
)
temporary_path = Path(temporary)
try:
os.chmod(temporary_path, 0o600)
with os.fdopen(descriptor, "w", encoding="utf-8") as stream:
descriptor = -1
json.dump(state, stream, separators=(",", ":"))
stream.write("\n")
stream.flush()
os.fsync(stream.fileno())
os.replace(temporary_path, state_path)
except BaseException:
if descriptor >= 0:
os.close(descriptor)
temporary_path.unlink(missing_ok=True)
raise
elif not state_path.is_file():
raise LoaderError(
"runtime secrets are not synchronized; restart han-secrets systemd unit"
)
else:
state = load_json(state_path)
if (
state.get("version") != 1
or state.get("source") != source
or state.get("loader_config") != str(config_path.resolve())
):
raise LoaderError(
"runtime secret state does not match selected source/config; synchronize first"
)
if not manifest.is_file():
raise LoaderError("runtime secret manifest was not materialized")
manifest_entries: dict[str, str] = {}
for line in manifest.read_text(encoding="utf-8").splitlines():
key, value_path = line.split("=", 1)
manifest_entries[key] = value_path
specs = document.get("secrets")
if not isinstance(specs, dict) or set(manifest_entries) != set(specs):
raise LoaderError("runtime secret manifest does not match loader configuration")
child = dict(environ)
child["HAN_SECRETS_ACTIVE"] = "1"
child["HAN_RUNTIME_SECRET_DIR"] = str(runtime_dir)
child["HAN_RUNTIME_SECRET_MANIFEST"] = str(manifest)
for key, value_path in manifest_entries.items():
if not Path(value_path).is_file():
raise LoaderError("runtime secret manifest references a missing file")
child[f"{key}_FILE"] = value_path
if synchronize or consumers:
print(
f"han-secrets: synchronized {len(consumers)} service scope(s) from {source}",
file=sys.stderr,
)
return child
def main(argv: list[str] | None = None) -> int:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("action", choices=("sync", "run"))
parser.add_argument("--config", type=Path, default=Path(".env"))
parser.add_argument("--loader-config", type=Path)
arguments, command = parser.parse_known_args(argv)
if command and command[0] == "--":
command.pop(0)
if arguments.action == "run" and not command:
parser.error("run requires a command after --")
if arguments.action == "sync" and command:
parser.error("sync does not accept a command")
environment = dict(os.environ)
try:
public = load_public_config(arguments.config)
source, loader_config = loader_config_path(
public, arguments.loader_config, environment
)
child = prepare_environment(
loader_config,
source,
environment,
synchronize=arguments.action == "sync",
)
except (LoaderError, OSError, ValueError, json.JSONDecodeError) as exc:
message = str(exc) if isinstance(exc, LoaderError) else exc.__class__.__name__
print(f"han-secrets: {message}", file=sys.stderr)
return 1
if arguments.action == "sync":
return 0
if os.name == "nt":
return subprocess.call(command, env=child)
os.execvpe(command[0], command, child)
return 127
if __name__ == "__main__":
raise SystemExit(main())
VM1_app/codebase/backend/deployment/secrets/han-secrets@.service
+42
View File
@@ -0,0 +1,42 @@
[Unit]
Description=Materialize HAN service secrets (%i)
Documentation=file:/usr/local/share/doc/han-secrets/SELECTEL_RUNBOOK.ru.md
Wants=network-online.target
After=network-online.target
Before=han-stack@%i.service
[Service]
Type=oneshot
User=root
Group=root
UMask=0077
RuntimeDirectory=han-chat/secrets
RuntimeDirectoryMode=0700
ExecStart=/usr/bin/python3 /usr/local/lib/han-secrets/han-secrets sync --config /opt/han-chat/backend/.env
LoadCredentialEncrypted=selectel-service-user-password:/etc/han/credentials/%i.selectel-password.cred
RemainAfterExit=yes
StandardOutput=null
StandardError=journal
SyslogIdentifier=han-secrets-%i
NoNewPrivileges=yes
PrivateTmp=yes
PrivateDevices=yes
ProtectSystem=strict
ProtectHome=yes
ProtectKernelTunables=yes
ProtectKernelModules=yes
ProtectKernelLogs=yes
ProtectControlGroups=yes
ProtectClock=yes
RestrictRealtime=yes
RestrictSUIDSGID=yes
LockPersonality=yes
MemoryDenyWriteExecute=yes
LimitCORE=0
SystemCallArchitectures=native
RestrictAddressFamilies=AF_UNIX AF_INET AF_INET6
CapabilityBoundingSet=
AmbientCapabilities=
[Install]
WantedBy=multi-user.target
VM1_app/codebase/backend/deployment/secrets/secrets_loader.py
+682
View File
@@ -0,0 +1,682 @@
#!/usr/bin/env python3
"""Materialize narrowly scoped service dotenv files from Selectel Secrets Manager."""
from __future__ import annotations
import argparse
import base64
import binascii
import json
import os
import random
import re
import ssl
import stat
import sys
import tempfile
import time
import urllib.error
import urllib.parse
import urllib.request
from dataclasses import dataclass
from pathlib import Path
from typing import Any, Callable, Mapping, NoReturn
DEFAULT_IDENTITY_URL = "https://cloud.api.selcloud.ru/identity/v3/auth/tokens"
MAX_CONFIG_BYTES = 1_048_576
MAX_HTTP_BYTES = 1_048_576
MAX_SECRET_BYTES = 65_536
RETRYABLE_STATUS = frozenset({408, 425, 429, 500, 502, 503, 504})
ENV_NAME_RE = re.compile(r"^[A-Z][A-Z0-9_]*$")
SERVICE_NAME_RE = re.compile(r"^[a-zA-Z0-9][a-zA-Z0-9_.-]*$")
DOTENV_LINE_RE = re.compile(r"^([A-Z][A-Z0-9_]*)=(.*)$")
class LoaderError(Exception):
"""An expected, already-redacted loader failure."""
def fail(message: str) -> NoReturn:
raise LoaderError(message)
def _object(value: Any, label: str) -> dict[str, Any]:
if not isinstance(value, dict):
fail(f"{label} must be an object")
return value
def _only_keys(value: Mapping[str, Any], allowed: set[str], label: str) -> None:
unknown = sorted(set(value) - allowed)
if unknown:
fail(f"{label} contains unsupported fields: {', '.join(unknown)}")
def _required_string(value: Mapping[str, Any], key: str, label: str) -> str:
item = value.get(key)
if not isinstance(item, str) or not item:
fail(f"{label}.{key} must be a non-empty string")
return item
def _bounded_int(value: Any, label: str, minimum: int, maximum: int) -> int:
if isinstance(value, bool) or not isinstance(value, int) or not minimum <= value <= maximum:
fail(f"{label} must be an integer from {minimum} through {maximum}")
return value
def read_limited(path: Path, limit: int, label: str) -> bytes:
try:
with path.open("rb") as stream:
data = stream.read(limit + 1)
except OSError as exc:
fail(f"cannot read {label}: {exc.strerror or exc.__class__.__name__}")
if len(data) > limit:
fail(f"{label} exceeds {limit} bytes")
return data
def require_private_regular_file(path: Path, label: str) -> None:
try:
metadata = path.lstat()
except OSError as exc:
fail(f"cannot inspect {label}: {exc.strerror or exc.__class__.__name__}")
if stat.S_ISLNK(metadata.st_mode) or not stat.S_ISREG(metadata.st_mode):
fail(f"{label} must be a regular file and not a symlink")
if os.name != "nt" and stat.S_IMODE(metadata.st_mode) & 0o077:
fail(f"{label} must not be accessible by group or other users")
def load_json(path: Path) -> dict[str, Any]:
raw = read_limited(path, MAX_CONFIG_BYTES, "configuration")
try:
document = json.loads(raw.decode("utf-8"))
except (UnicodeDecodeError, json.JSONDecodeError):
fail("configuration is not valid UTF-8 JSON")
return _object(document, "configuration")
def credential_value(selectel: Mapping[str, Any], environ: Mapping[str, str]) -> str:
methods = sum(key in selectel for key in ("password_file", "password_env"))
if methods != 1:
fail("selectel must set exactly one of password_file or password_env")
if "password_env" in selectel:
variable = _required_string(selectel, "password_env", "selectel")
if not ENV_NAME_RE.fullmatch(variable):
fail("selectel.password_env is not a valid environment variable name")
value = environ.get(variable)
if value is None or not value:
fail(f"credential environment variable {variable} is not set")
return value
configured = Path(_required_string(selectel, "password_file", "selectel"))
if configured.is_absolute():
path = configured
else:
directory = environ.get("CREDENTIALS_DIRECTORY")
if not directory:
fail("relative password_file requires CREDENTIALS_DIRECTORY")
path = Path(directory) / configured
require_private_regular_file(path, "credential")
raw = read_limited(path, 16_384, "credential")
try:
value = raw.decode("utf-8")
except UnicodeDecodeError:
fail("credential is not valid UTF-8")
value = value.removesuffix("\n").removesuffix("\r")
if not value or "\n" in value or "\r" in value or "\x00" in value:
fail("credential must contain exactly one non-empty text line")
return value
class NoRedirect(urllib.request.HTTPRedirectHandler):
def redirect_request(self, req: Any, fp: Any, code: int, msg: str, headers: Any, newurl: str) -> None:
return None
@dataclass(frozen=True)
class HTTPResult:
status: int
headers: Mapping[str, str]
body: bytes
class HTTPClient:
def __init__(
self,
*,
timeout: float,
retries: int,
max_response_bytes: int,
cafile: str | None = None,
opener: Any | None = None,
sleeper: Callable[[float], None] = time.sleep,
jitter: Callable[[], float] = random.random,
) -> None:
self.timeout = timeout
self.retries = retries
self.max_response_bytes = max_response_bytes
self.sleeper = sleeper
self.jitter = jitter
if opener is None:
try:
context = ssl.create_default_context(cafile=cafile)
except (OSError, ssl.SSLError) as exc:
fail(f"cannot initialize TLS trust store: {exc.__class__.__name__}")
self.opener = urllib.request.build_opener(
urllib.request.HTTPSHandler(context=context), NoRedirect()
)
else:
self.opener = opener
def request(
self,
method: str,
url: str,
*,
headers: Mapping[str, str] | None = None,
body: bytes | None = None,
expected: frozenset[int],
) -> HTTPResult:
parsed = urllib.parse.urlsplit(url)
if parsed.scheme != "https" or not parsed.netloc or parsed.username or parsed.password:
fail("provider endpoint must be an HTTPS URL without embedded credentials")
request = urllib.request.Request(
url, data=body, headers=dict(headers or {}), method=method
)
for attempt in range(self.retries + 1):
try:
with self.opener.open(request, timeout=self.timeout) as response:
status = int(response.status)
content_length = response.headers.get("Content-Length")
if content_length:
try:
if int(content_length) > self.max_response_bytes:
fail("provider response exceeds configured limit")
except ValueError:
fail("provider returned an invalid Content-Length")
response_body = response.read(self.max_response_bytes + 1)
if len(response_body) > self.max_response_bytes:
fail("provider response exceeds configured limit")
if status not in expected:
fail(f"provider request failed with HTTP {status}")
return HTTPResult(status, response.headers, response_body)
except urllib.error.HTTPError as exc:
status = int(exc.code)
if status not in RETRYABLE_STATUS or attempt >= self.retries:
fail(f"provider request failed with HTTP {status}")
except (urllib.error.URLError, TimeoutError, OSError):
if attempt >= self.retries:
fail("provider request failed after retries")
delay = min(8.0, 0.25 * (2**attempt)) * (0.5 + self.jitter())
self.sleeper(delay)
fail("provider request failed")
def parse_json_response(result: HTTPResult, label: str) -> dict[str, Any]:
try:
return _object(json.loads(result.body.decode("utf-8")), label)
except (UnicodeDecodeError, json.JSONDecodeError):
fail(f"{label} is not valid JSON")
def project_token_and_catalog(
client: HTTPClient, selectel: Mapping[str, Any], password: str
) -> tuple[str, list[Any]]:
identity_url = selectel.get("identity_url", DEFAULT_IDENTITY_URL)
if not isinstance(identity_url, str):
fail("selectel.identity_url must be a string")
account_id = _required_string(selectel, "account_id", "selectel")
username = _required_string(selectel, "username", "selectel")
project_name = _required_string(selectel, "project_name", "selectel")
payload = {
"auth": {
"identity": {
"methods": ["password"],
"password": {
"user": {
"name": username,
"domain": {"name": account_id},
"password": password,
}
},
},
"scope": {
"project": {
"name": project_name,
"domain": {"name": account_id},
}
},
}
}
result = client.request(
"POST",
identity_url,
headers={"Content-Type": "application/json", "Accept": "application/json"},
body=json.dumps(payload, separators=(",", ":")).encode("utf-8"),
expected=frozenset({201}),
)
token = result.headers.get("X-Subject-Token")
if not isinstance(token, str) or not token:
fail("identity response omitted X-Subject-Token")
document = parse_json_response(result, "identity response")
token_data = document.get("token")
if not isinstance(token_data, dict):
fail("identity response omitted token metadata")
project = token_data.get("project")
if not isinstance(project, dict) or not project.get("id"):
fail("identity token is not project-scoped")
catalog = token_data.get("catalog")
if not isinstance(catalog, list):
fail("identity response omitted service catalog")
return token, catalog
def secrets_endpoint(catalog: list[Any], region: str, interface: str) -> str:
matches: list[str] = []
for service in catalog:
if not isinstance(service, dict) or service.get("type") != "secrets-manager":
continue
endpoints = service.get("endpoints")
if not isinstance(endpoints, list):
continue
for endpoint in endpoints:
if (
isinstance(endpoint, dict)
and endpoint.get("region") == region
and endpoint.get("interface") == interface
and isinstance(endpoint.get("url"), str)
):
matches.append(endpoint["url"].rstrip("/"))
if len(matches) != 1:
fail("service catalog did not contain exactly one matching Secrets Manager endpoint")
return matches[0]
def decode_secret(document: Mapping[str, Any], name: str, limit: int) -> bytes:
# GET /v1/{name} returns the current value inside ``version`` while
# GET /v1/{name}/versions/{id} returns a version object directly.
payload: Mapping[str, Any] = document
version = document.get("version")
if isinstance(version, dict):
payload = version
encoded = payload.get("value")
if not isinstance(encoded, str):
fail(f"secret {name} response omitted base64 value")
try:
value = base64.b64decode(encoded, validate=True)
except (binascii.Error, ValueError):
fail(f"secret {name} has invalid base64 encoding")
if not value:
fail(f"secret {name} is empty")
if len(value) > limit:
fail(f"secret {name} exceeds its configured limit")
if b"\x00" in value or b"\n" in value or b"\r" in value:
fail(f"secret {name} cannot be represented as a dotenv value")
return value
def fetch_selectel(
config: Mapping[str, Any],
specs: Mapping[str, Mapping[str, Any]],
environ: Mapping[str, str],
client_factory: Callable[..., HTTPClient] = HTTPClient,
) -> dict[str, bytes]:
selectel = _object(config.get("selectel"), "selectel")
_only_keys(
selectel,
{
"account_id",
"username",
"project_name",
"region",
"interface",
"password_file",
"password_env",
"identity_url",
"secrets_url",
"ca_file",
},
"selectel",
)
http = _object(config.get("http", {}), "http")
_only_keys(http, {"timeout_seconds", "retries", "max_response_bytes"}, "http")
timeout = http.get("timeout_seconds", 10)
if isinstance(timeout, bool) or not isinstance(timeout, (int, float)) or not 0.1 <= timeout <= 60:
fail("http.timeout_seconds must be from 0.1 through 60")
retries = _bounded_int(http.get("retries", 3), "http.retries", 0, 8)
response_limit = _bounded_int(
http.get("max_response_bytes", MAX_HTTP_BYTES),
"http.max_response_bytes",
1024,
4 * MAX_HTTP_BYTES,
)
cafile = selectel.get("ca_file")
if cafile is not None and (not isinstance(cafile, str) or not cafile):
fail("selectel.ca_file must be a non-empty string")
client = client_factory(
timeout=float(timeout),
retries=retries,
max_response_bytes=response_limit,
cafile=cafile,
)
password = credential_value(selectel, environ)
token, catalog = project_token_and_catalog(client, selectel, password)
region = _required_string(selectel, "region", "selectel")
interface = selectel.get("interface", "public")
if interface not in {"public", "internal"}:
fail("selectel.interface must be public or internal")
override = selectel.get("secrets_url")
if override is not None and (not isinstance(override, str) or not override):
fail("selectel.secrets_url must be a non-empty string")
base_url = override.rstrip("/") if override else secrets_endpoint(catalog, region, interface)
values: dict[str, bytes] = {}
fetched: dict[tuple[str, int | None], bytes] = {}
for canonical, spec in specs.items():
if "literal" in spec:
values[canonical] = b""
continue
remote = _required_string(spec, "remote", f"secrets.{canonical}")
version = spec.get("version")
version_id: int | None = None
if version is not None:
version_id = _bounded_int(
version, f"secrets.{canonical}.version", 1, 2_147_483_647
)
cache_key = (remote, version_id)
if cache_key not in fetched:
path = f"/v1/{urllib.parse.quote(remote, safe='')}"
if version_id is not None:
path += f"/versions/{version_id}"
try:
result = client.request(
"GET",
base_url + path,
headers={"X-Auth-Token": token, "Accept": "application/json"},
expected=frozenset({200}),
)
document = parse_json_response(result, f"secret {canonical} response")
fetched[cache_key] = decode_secret(
document, canonical, MAX_SECRET_BYTES
)
except LoaderError as exc:
fail(f"cannot load {canonical}: {exc}")
limit = _bounded_int(
spec.get("max_bytes", MAX_SECRET_BYTES),
f"secrets.{canonical}.max_bytes",
1,
MAX_SECRET_BYTES,
)
value = fetched[cache_key]
if len(value) > limit:
fail(f"secret {canonical} exceeds its configured limit")
values[canonical] = value
return values
def parse_dotenv(path: Path, expected: set[str], max_bytes: int) -> dict[str, bytes]:
require_private_regular_file(path, "fallback dotenv")
raw = read_limited(path, max_bytes, "fallback dotenv")
try:
text = raw.decode("utf-8")
except UnicodeDecodeError:
fail("fallback dotenv is not valid UTF-8")
values: dict[str, bytes] = {}
for number, line in enumerate(text.splitlines(), 1):
if not line or line.startswith("#"):
continue
match = DOTENV_LINE_RE.fullmatch(line)
if not match:
fail(f"fallback dotenv has invalid syntax at line {number}")
name, encoded_value = match.groups()
if name not in expected:
fail(f"fallback dotenv contains undeclared key {name}")
if name in values:
fail(f"fallback dotenv contains duplicate key {name}")
if encoded_value.startswith('"'):
try:
decoded = json.loads(encoded_value)
except json.JSONDecodeError:
fail(f"fallback dotenv has invalid quoted value at line {number}")
if not isinstance(decoded, str):
fail(f"fallback dotenv has invalid quoted value at line {number}")
value = decoded.encode("utf-8")
elif encoded_value.startswith("'"):
if len(encoded_value) < 2 or not encoded_value.endswith("'"):
fail(f"fallback dotenv has invalid quoted value at line {number}")
value = encoded_value[1:-1].encode("utf-8")
else:
if any(character.isspace() for character in encoded_value) or any(
character in encoded_value for character in ("'", '"', "`", "$", "\\")
):
fail(f"fallback dotenv requires quoting at line {number}")
value = encoded_value.encode("utf-8")
if not value or b"\x00" in value or b"\n" in value or b"\r" in value:
fail(f"fallback dotenv has an empty or unsafe value for {name}")
values[name] = value
missing = sorted(expected - set(values))
if missing:
fail(f"fallback dotenv is missing declared keys: {', '.join(missing)}")
return values
def validate_specs(config: Mapping[str, Any]) -> dict[str, dict[str, Any]]:
raw_specs = _object(config.get("secrets"), "secrets")
if not raw_specs:
fail("secrets must not be empty")
specs: dict[str, dict[str, Any]] = {}
for canonical, raw_spec in raw_specs.items():
if not isinstance(canonical, str) or not ENV_NAME_RE.fullmatch(canonical):
fail("every canonical secret name must be an uppercase environment name")
spec = _object(raw_spec, f"secrets.{canonical}")
_only_keys(
spec,
{"remote", "consumers", "max_bytes", "version", "literal"},
f"secrets.{canonical}",
)
has_remote = "remote" in spec
has_literal = "literal" in spec
if has_remote == has_literal:
fail(f"secrets.{canonical} must set exactly one of remote or literal")
if has_literal and spec["literal"] != "":
fail(f"secrets.{canonical}.literal may only be an empty string")
consumers = spec.get("consumers")
if not isinstance(consumers, list) or not consumers:
fail(f"secrets.{canonical}.consumers must be a non-empty array")
if len(consumers) != len(set(item for item in consumers if isinstance(item, str))):
fail(f"secrets.{canonical}.consumers contains duplicates or invalid values")
for consumer in consumers:
if not isinstance(consumer, str) or not SERVICE_NAME_RE.fullmatch(consumer):
fail(f"secrets.{canonical}.consumers contains an invalid service name")
specs[canonical] = spec
return specs
def dotenv_quote(value: bytes, name: str) -> str:
try:
text = value.decode("utf-8")
except UnicodeDecodeError:
fail(f"secret {name} is not valid UTF-8")
return json.dumps(text, ensure_ascii=False)
def materialize(runtime_dir: Path, specs: Mapping[str, Mapping[str, Any]], values: Mapping[str, bytes]) -> None:
try:
runtime_dir.mkdir(mode=0o700, parents=True, exist_ok=True)
if runtime_dir.is_symlink():
fail("runtime directory must not be a symlink")
os.chmod(runtime_dir, 0o700)
except OSError as exc:
fail(f"cannot prepare runtime directory: {exc.strerror or exc.__class__.__name__}")
consumers = sorted(
{consumer for spec in specs.values() for consumer in spec["consumers"]}
)
staged: list[tuple[Path, Path]] = []
try:
for consumer in consumers:
lines = [
f"{name}={dotenv_quote(values[name], name)}\n"
for name, spec in sorted(specs.items())
if consumer in spec["consumers"]
]
descriptor, temporary = tempfile.mkstemp(
prefix=f".{consumer}.", suffix=".tmp", dir=runtime_dir
)
temporary_path = Path(temporary)
try:
os.chmod(temporary_path, 0o600)
stream = os.fdopen(descriptor, "w", encoding="utf-8", newline="\n")
descriptor = -1
with stream:
stream.writelines(lines)
stream.flush()
os.fsync(stream.fileno())
except BaseException:
if descriptor >= 0:
os.close(descriptor)
raise
staged.append((temporary_path, runtime_dir / f"{consumer}.env"))
for temporary_path, destination in staged:
os.replace(temporary_path, destination)
value_paths: dict[str, Path] = {}
for name, value in sorted(values.items()):
descriptor, temporary = tempfile.mkstemp(
prefix=f".{name}.", suffix=".tmp", dir=runtime_dir
)
temporary_path = Path(temporary)
try:
# Compose implements local secrets as bind mounts. The protected
# 0700 parent prevents host users from traversing to this 0444
# file while allowing a non-root container UID to read its mount.
os.chmod(temporary_path, 0o444)
with os.fdopen(descriptor, "wb") as stream:
descriptor = -1
stream.write(value)
stream.flush()
os.fsync(stream.fileno())
except BaseException:
if descriptor >= 0:
os.close(descriptor)
temporary_path.unlink(missing_ok=True)
raise
destination = runtime_dir / name
os.replace(temporary_path, destination)
value_paths[name] = destination
manifest_lines = [
f"{name}={path.resolve()}\n" for name, path in sorted(value_paths.items())
]
descriptor, temporary = tempfile.mkstemp(
prefix=".manifest.", suffix=".tmp", dir=runtime_dir
)
manifest_path = Path(temporary)
try:
os.chmod(manifest_path, 0o600)
with os.fdopen(descriptor, "w", encoding="utf-8", newline="\n") as stream:
descriptor = -1
stream.writelines(manifest_lines)
stream.flush()
os.fsync(stream.fileno())
except BaseException:
if descriptor >= 0:
os.close(descriptor)
manifest_path.unlink(missing_ok=True)
raise
os.replace(manifest_path, runtime_dir / "manifest")
if hasattr(os, "O_DIRECTORY"):
directory_fd = os.open(runtime_dir, os.O_RDONLY | os.O_DIRECTORY)
try:
os.fsync(directory_fd)
finally:
os.close(directory_fd)
except OSError as exc:
fail(f"cannot atomically materialize service files: {exc.strerror or exc.__class__.__name__}")
finally:
for temporary_path, _ in staged:
try:
temporary_path.unlink(missing_ok=True)
except OSError:
pass
def run(
config_path: Path,
*,
runtime_override: Path | None = None,
environ: Mapping[str, str] | None = None,
client_factory: Callable[..., HTTPClient] = HTTPClient,
) -> list[str]:
os.umask(0o077)
environment = os.environ if environ is None else environ
config = load_json(config_path)
_only_keys(config, {"version", "mode", "runtime_dir", "http", "selectel", "file", "secrets"}, "configuration")
if config.get("version") != 1:
fail("configuration.version must be 1")
mode = config.get("mode")
if mode not in {"selectel", "file"}:
fail("configuration.mode must explicitly be selectel or file")
specs = validate_specs(config)
if runtime_override is None:
configured_runtime = config.get("runtime_dir")
if not isinstance(configured_runtime, str) or not configured_runtime:
fail("configuration.runtime_dir must be a non-empty string")
runtime_dir = Path(configured_runtime)
else:
runtime_dir = runtime_override
if not runtime_dir.is_absolute():
fail("runtime directory must be an absolute path")
if mode == "selectel":
if "file" in config:
fail("file settings are forbidden in selectel mode")
values = fetch_selectel(config, specs, environment, client_factory)
else:
if "selectel" in config or "http" in config:
fail("selectel and http settings are forbidden in file mode")
file_config = _object(config.get("file"), "file")
_only_keys(file_config, {"path", "max_bytes"}, "file")
source = Path(_required_string(file_config, "path", "file"))
if not source.is_absolute():
fail("file.path must be absolute")
max_bytes = _bounded_int(
file_config.get("max_bytes", MAX_CONFIG_BYTES),
"file.max_bytes",
1,
4 * MAX_CONFIG_BYTES,
)
expected = {name for name, spec in specs.items() if "literal" not in spec}
values = parse_dotenv(source, expected, max_bytes)
values.update(
{name: b"" for name, spec in specs.items() if "literal" in spec}
)
for canonical, value in values.items():
limit = _bounded_int(
specs[canonical].get("max_bytes", MAX_SECRET_BYTES),
f"secrets.{canonical}.max_bytes",
1,
MAX_SECRET_BYTES,
)
if len(value) > limit:
fail(f"secret {canonical} exceeds its configured limit")
materialize(runtime_dir, specs, values)
return sorted({consumer for spec in specs.values() for consumer in spec["consumers"]})
def main(argv: list[str] | None = None) -> int:
parser = argparse.ArgumentParser(description="Materialize per-service secret dotenv files")
parser.add_argument("--config", required=True, type=Path)
parser.add_argument("--runtime-dir", type=Path)
arguments = parser.parse_args(argv)
try:
consumers = run(arguments.config, runtime_override=arguments.runtime_dir)
except LoaderError as exc:
print(f"secrets-loader: {exc}", file=sys.stderr)
return 1
print(f"secrets-loader: materialized {len(consumers)} service file(s)", file=sys.stderr)
return 0
if __name__ == "__main__":
raise SystemExit(main())
VM1_app/codebase/backend/docker-compose.yml
+28
View File
@@ -0,0 +1,28 @@
name: han-chat
include:
- path: ./infra/compose/application.yml
- path: ./nginx/docker-compose.yml
- path: ./redis/docker-compose.yml
- path: ./observability/docker-compose.yml
- path: ./deployment/docker-compose.jobs.yml
networks:
public:
name: han-chat-public
backend:
name: han-chat-backend
internal: true
egress:
name: han-chat-egress
observability:
name: han-chat-observability
internal: true
volumes:
redis-data:
nginx-certs:
nginx-acme-webroot:
nginx-cache:
frontend-static:
otel-queue:

Some files were not shown because too many files have changed in this diff Show More