Проект разделен на два репозитория
This commit is contained in:
@@ -0,0 +1,16 @@
|
||||
BITRIX_SYNC_ENABLED=false
|
||||
BITRIX_SYNC_MODE=disabled
|
||||
BITRIX_SYNC_DATABASE_URL_FILE=/run/secrets/bitrix_sync_database_url
|
||||
BITRIX_SYNC_CRM_REST_WEBHOOK_URL_FILE=/run/secrets/bitrix_sync_crm_url
|
||||
BITRIX_SYNC_CONTACT_RECEIVER_TOKEN_FILE=/run/secrets/bitrix_sync_contact_token
|
||||
BITRIX_SYNC_ALERT_RECEIVER_TOKEN_FILE=/run/secrets/bitrix_sync_alert_token
|
||||
BITRIX_SYNC_SERVICE_TOKEN_FILE=/run/secrets/bitrix_sync_service_token
|
||||
BITRIX_SYNC_PORTAL_HOST=portal.example.bitrix24.ru
|
||||
BITRIX_SYNC_PORTAL_MEMBER_ID=replace-with-member-id
|
||||
BITRIX_SYNC_PUBLIC_BASE_URL=https://sync.example.ru
|
||||
BITRIX_SYNC_CONTACT_USER_ID_FIELD=UF_CRM_100
|
||||
BITRIX_SYNC_CONTACT_REGISTERED_FIELD=UF_CRM_101
|
||||
BITRIX_SYNC_CONTACT_CITIZENSHIP_FIELD=UF_CRM_102
|
||||
BITRIX_SYNC_WEBHOOK_ALLOWED_CIDRS=203.0.113.0/24
|
||||
BITRIX_SYNC_HTTP_TIMEOUT_SEC=10
|
||||
BITRIX_SYNC_DB_POOL_SIZE=5
|
||||
@@ -0,0 +1,18 @@
|
||||
ARG PYTHON_IMAGE=python:3.12.11-slim
|
||||
FROM ${PYTHON_IMAGE} AS build
|
||||
WORKDIR /build
|
||||
COPY pyproject.toml ./
|
||||
COPY app ./app
|
||||
RUN pip install --no-cache-dir --prefix=/install .
|
||||
|
||||
FROM ${PYTHON_IMAGE}
|
||||
ENV PYTHONDONTWRITEBYTECODE=1 PYTHONUNBUFFERED=1 PATH="/opt/venv/bin:${PATH}"
|
||||
RUN groupadd --gid 10001 han && useradd --uid 10001 --gid 10001 --no-create-home han
|
||||
COPY --from=build /install /usr/local
|
||||
COPY --chown=10001:10001 app /srv/app
|
||||
COPY --chown=10001:10001 alembic /srv/alembic
|
||||
COPY --chown=10001:10001 alembic.ini openapi.yaml /srv/
|
||||
WORKDIR /srv
|
||||
USER 10001:10001
|
||||
EXPOSE 8080
|
||||
ENTRYPOINT ["han-bitrix-sync-api"]
|
||||
@@ -0,0 +1,72 @@
|
||||
# bitrix-sync
|
||||
|
||||
Изолированный Python 3.12 сервис durable-синхронизации Contact между `han_app` и
|
||||
Битрикс24. Сервис не участвует в Open Lines и не имеет HTTP-зависимости от
|
||||
`api-backend`.
|
||||
|
||||
## Entrypoints
|
||||
|
||||
- `han-bitrix-sync-api` — health, internal status и два bounded robot receiver;
|
||||
- `han-bitrix-sync-worker` — queue/webhook/rebind workflows с lease fencing;
|
||||
- `han-bitrix-sync-reconciliation` — один advisory-lock incremental run;
|
||||
- `alembic upgrade head` — отдельная контролируемая миграция, не startup DDL.
|
||||
|
||||
Disabled mode требует только `BITRIX_SYNC_ENABLED=false` и
|
||||
`BITRIX_SYNC_MODE=disabled`, не читает БД и не принимает webhook. Full mode
|
||||
валидирует весь каталог secret files, portal identity, custom fields, HTTPS host
|
||||
lock и непустой CIDR allow-list до startup.
|
||||
|
||||
## Границы безопасности
|
||||
|
||||
- CRM credential URL используется как единый секрет; redirect выключен, TLS
|
||||
проверяется, REST method выбирается только из закрытого allow-list.
|
||||
- Receiver принимает только `application/x-www-form-urlencoded` с bounded
|
||||
content length, числом и длиной полей. Query token сравнивается constant-time.
|
||||
- nginx должен перезаписывать `X-Real-IP` из TCP peer, проверять CIDR до proxy и
|
||||
не логировать `$request_uri`, args или body. Контейнер receiver недоступен
|
||||
напрямую.
|
||||
- DB хранит только hash CRM-master значений в snapshot; safe command projection
|
||||
не содержит PII. URL credential, form body и token не логируются.
|
||||
- Запись CRM-master полей выполняется в одной транзакции после
|
||||
`SET LOCAL han.sync_suppress='true'`.
|
||||
|
||||
## Локальные проверки
|
||||
|
||||
Лёгкие проверки, не требующие Docker, сервиса или реального PostgreSQL:
|
||||
|
||||
```text
|
||||
python -m pytest
|
||||
python -m ruff check app tests
|
||||
```
|
||||
|
||||
PostgreSQL integration и Bitrix contract suites намеренно являются внешними
|
||||
gates: локальный managed PostgreSQL не поднимается Compose-файлом.
|
||||
|
||||
## External gates до `BITRIX_SYNC_ENABLED=true`
|
||||
|
||||
1. Применить migrations migration-role и проверить grants runtime-role.
|
||||
2. На disposable managed PostgreSQL проверить concurrent `SKIP LOCKED`,
|
||||
lease expiry/fencing, active mapping uniqueness, rebind partial failure,
|
||||
transaction-local GUC без утечки и crash после CRM success.
|
||||
3. Подтвердить на целевом портале wire-контракты `duplicate.findbycomm`,
|
||||
Contact add/get/update, mixed `batch`, custom fields, enum dictionary и
|
||||
`crm.item.list` с `opened=1`, registration REST field `=1`.
|
||||
4. Заполнить и активировать валидную `business_alerts` settings version:
|
||||
entity/category/stage/field IDs и responsible party. Placeholder `null`
|
||||
запрещает alert receiver.
|
||||
5. Проверить least-privilege credential negative tests; credential администратора
|
||||
запрещён.
|
||||
6. Валидировать nginx exact routes, no-redirect HTTP policy, body/rate limits,
|
||||
version-controlled CIDR и отсутствие query/body в access/error/traces.
|
||||
7. Запустить synthetic webhook с реальным robot form contract, затем убедиться,
|
||||
что durable inbox commit предшествует `202`.
|
||||
8. Выполнить 10k incremental reconciliation/load gate, webhook-loss recovery,
|
||||
429/5xx/TLS/DNS/timeout/uncertain-create и restart-at-each-step tests.
|
||||
9. Проверить container image digest, dependency/SBOM/vulnerability scan и
|
||||
compose hardening; root Compose ВМ2 подключает этот fragment отдельно.
|
||||
10. Зафиксировать cutover watermark, отменить только pre-cutover active tasks,
|
||||
выполнить disabled preflight и затем controlled enablement.
|
||||
|
||||
`compose.fragment.yaml` — сервисный фрагмент, не root Compose и не команда
|
||||
развёртывания. Reconciliation entrypoint выполняет один run; расписание задаёт
|
||||
root-owned scheduler/deployment layer.
|
||||
@@ -0,0 +1,30 @@
|
||||
[alembic]
|
||||
script_location = alembic
|
||||
prepend_sys_path = .
|
||||
sqlalchemy.url = postgresql+asyncpg://unused
|
||||
|
||||
[loggers]
|
||||
keys = root,sqlalchemy,alembic
|
||||
[handlers]
|
||||
keys = console
|
||||
[formatters]
|
||||
keys = generic
|
||||
[logger_root]
|
||||
level = WARN
|
||||
handlers = console
|
||||
qualname =
|
||||
[logger_sqlalchemy]
|
||||
level = WARN
|
||||
handlers =
|
||||
qualname = sqlalchemy.engine
|
||||
[logger_alembic]
|
||||
level = INFO
|
||||
handlers =
|
||||
qualname = alembic
|
||||
[handler_console]
|
||||
class = StreamHandler
|
||||
args = (sys.stderr,)
|
||||
level = NOTSET
|
||||
formatter = generic
|
||||
[formatter_generic]
|
||||
format = %(levelname)-5.5s [%(name)s] %(message)s
|
||||
@@ -0,0 +1,58 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
from sqlalchemy import pool
|
||||
from sqlalchemy.ext.asyncio import async_engine_from_config
|
||||
|
||||
from alembic import context
|
||||
from app.repository import postgres_ssl_context
|
||||
|
||||
|
||||
def migration_url() -> str:
|
||||
path = os.getenv("BITRIX_SYNC_MIGRATION_DATABASE_URL_FILE")
|
||||
if not path:
|
||||
raise RuntimeError("BITRIX_SYNC_MIGRATION_DATABASE_URL_FILE is required")
|
||||
value = Path(path).read_text(encoding="utf-8").rstrip("\r\n")
|
||||
if not value:
|
||||
raise RuntimeError("Bitrix migration database URL file is empty")
|
||||
return value
|
||||
|
||||
|
||||
def run_migrations_offline() -> None:
|
||||
context.configure(
|
||||
url=migration_url(),
|
||||
target_metadata=None,
|
||||
literal_binds=True,
|
||||
dialect_opts={"paramstyle": "named"},
|
||||
)
|
||||
with context.begin_transaction():
|
||||
context.run_migrations()
|
||||
|
||||
|
||||
async def run_async_migrations() -> None:
|
||||
configuration = context.config.get_section(context.config.config_ini_section) or {}
|
||||
configuration["sqlalchemy.url"] = migration_url()
|
||||
engine = async_engine_from_config(
|
||||
configuration,
|
||||
prefix="sqlalchemy.",
|
||||
poolclass=pool.NullPool,
|
||||
connect_args={"ssl": postgres_ssl_context()},
|
||||
)
|
||||
|
||||
def run_sync_migrations(connection) -> None:
|
||||
context.configure(connection=connection, target_metadata=None, compare_type=True)
|
||||
with context.begin_transaction():
|
||||
context.run_migrations()
|
||||
|
||||
async with engine.connect() as connection:
|
||||
await connection.run_sync(run_sync_migrations)
|
||||
await engine.dispose()
|
||||
|
||||
|
||||
if context.is_offline_mode():
|
||||
run_migrations_offline()
|
||||
else:
|
||||
asyncio.run(run_async_migrations())
|
||||
+20
@@ -0,0 +1,20 @@
|
||||
"""Preserve the legacy connectivity-stub Alembic revision.
|
||||
|
||||
Revision ID: 0001_sync_baseline
|
||||
Revises:
|
||||
"""
|
||||
|
||||
from collections.abc import Sequence
|
||||
|
||||
revision: str = "0001_sync_baseline"
|
||||
down_revision: str | None = None
|
||||
branch_labels: str | Sequence[str] | None = None
|
||||
depends_on: str | Sequence[str] | None = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
"""The legacy connectivity stub owned no runtime tables."""
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
"""The legacy connectivity stub owned no runtime tables."""
|
||||
@@ -0,0 +1,356 @@
|
||||
"""Create durable bitrix_sync schema and contracts.
|
||||
|
||||
Revision ID: 0001_bitrix_sync_full
|
||||
Revises: 0001_sync_baseline
|
||||
Create Date: 2026-08-06
|
||||
"""
|
||||
|
||||
from collections.abc import Sequence
|
||||
|
||||
from alembic import op
|
||||
|
||||
revision: str = "0001_bitrix_sync_full"
|
||||
down_revision: str | None = "0001_sync_baseline"
|
||||
branch_labels: str | Sequence[str] | None = None
|
||||
depends_on: str | Sequence[str] | None = None
|
||||
|
||||
|
||||
def _execute_script(script: str) -> None:
|
||||
"""Execute simple DDL statements separately for asyncpg compatibility."""
|
||||
for statement in script.split(";"):
|
||||
if statement.strip():
|
||||
op.execute(statement)
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.execute("CREATE EXTENSION IF NOT EXISTS pgcrypto")
|
||||
op.execute("CREATE SCHEMA IF NOT EXISTS bitrix_sync")
|
||||
_execute_script(
|
||||
"""
|
||||
CREATE TABLE bitrix_sync.workflow_instances (
|
||||
id uuid PRIMARY KEY,
|
||||
workflow_type varchar(64) NOT NULL CHECK (workflow_type IN
|
||||
('contact.map_or_create','contact.update','contact.deactivate','contact.rebind',
|
||||
'contact.webhook','contact.reconciliation','alert.reconciliation')),
|
||||
user_id uuid,
|
||||
external_id varchar(128),
|
||||
state varchar(24) NOT NULL CHECK (state IN
|
||||
('created','running','waiting_crm','waiting_retry','waiting_manual',
|
||||
'succeeded','failed','cancelled')),
|
||||
current_step varchar(64) NOT NULL,
|
||||
source_task_id uuid UNIQUE,
|
||||
deadline_at timestamptz NOT NULL,
|
||||
outcome varchar(64),
|
||||
completed_at timestamptz,
|
||||
created_at timestamptz NOT NULL DEFAULT now(),
|
||||
updated_at timestamptz NOT NULL DEFAULT now()
|
||||
);
|
||||
CREATE INDEX ix_workflow_claim
|
||||
ON bitrix_sync.workflow_instances(state,updated_at)
|
||||
WHERE state IN ('created','running','waiting_crm','waiting_retry');
|
||||
|
||||
CREATE TABLE bitrix_sync.entity_external_mapping (
|
||||
id uuid PRIMARY KEY,
|
||||
entity_type varchar(64) NOT NULL,
|
||||
entity_id uuid NOT NULL,
|
||||
external_system varchar(32) NOT NULL DEFAULT 'bitrix24'
|
||||
CHECK (external_system='bitrix24'),
|
||||
external_entity_type varchar(32) NOT NULL DEFAULT 'contact'
|
||||
CHECK (external_entity_type='contact'),
|
||||
external_id varchar(128) NOT NULL,
|
||||
status varchar(16) NOT NULL CHECK (status IN ('active','closed','broken')),
|
||||
opened_at timestamptz NOT NULL,
|
||||
closed_at timestamptz,
|
||||
close_reason varchar(64),
|
||||
workflow_id uuid REFERENCES bitrix_sync.workflow_instances(id),
|
||||
created_at timestamptz NOT NULL DEFAULT now(),
|
||||
updated_at timestamptz NOT NULL DEFAULT now(),
|
||||
CHECK ((status='active' AND closed_at IS NULL) OR
|
||||
(status IN ('closed','broken')
|
||||
AND (closed_at IS NOT NULL OR close_reason IS NOT NULL)))
|
||||
);
|
||||
CREATE UNIQUE INDEX uq_mapping_active_entity
|
||||
ON bitrix_sync.entity_external_mapping(external_system,entity_type,entity_id)
|
||||
WHERE status='active';
|
||||
CREATE UNIQUE INDEX uq_mapping_active_external
|
||||
ON bitrix_sync.entity_external_mapping
|
||||
(external_system,external_entity_type,external_id)
|
||||
WHERE status='active';
|
||||
CREATE INDEX ix_mapping_history
|
||||
ON bitrix_sync.entity_external_mapping(entity_id,opened_at DESC);
|
||||
|
||||
CREATE TABLE bitrix_sync.crm_commands (
|
||||
id uuid PRIMARY KEY,
|
||||
workflow_id uuid NOT NULL REFERENCES bitrix_sync.workflow_instances(id),
|
||||
command_type varchar(40) NOT NULL CHECK (command_type IN
|
||||
('duplicate_find','contact_get','contact_add','contact_update',
|
||||
'citizenship_fields_get','contact_incremental_list',
|
||||
'alert_get','alert_add','alert_update',
|
||||
'rebind_target_get','rebind_old_get')),
|
||||
safe_request jsonb NOT NULL DEFAULT '{}',
|
||||
status varchar(24) NOT NULL CHECK (status IN
|
||||
('pending','leased','in_flight','succeeded','retry','retry_wait',
|
||||
'uncertain','reconcile','dead_letter','permanent','rate_limited')),
|
||||
attempt_count integer NOT NULL DEFAULT 0 CHECK (attempt_count>=0),
|
||||
next_attempt_at timestamptz NOT NULL DEFAULT now(),
|
||||
locked_by varchar(128),
|
||||
locked_until timestamptz,
|
||||
lease_token uuid,
|
||||
batch_id uuid,
|
||||
correlation_id uuid,
|
||||
safe_response jsonb,
|
||||
safe_error_code varchar(64),
|
||||
http_status integer,
|
||||
completed_at timestamptz,
|
||||
created_at timestamptz NOT NULL DEFAULT now(),
|
||||
updated_at timestamptz NOT NULL DEFAULT now()
|
||||
);
|
||||
CREATE INDEX ix_crm_command_claim
|
||||
ON bitrix_sync.crm_commands(status,next_attempt_at,created_at);
|
||||
CREATE INDEX ix_crm_command_workflow
|
||||
ON bitrix_sync.crm_commands(workflow_id,created_at);
|
||||
|
||||
CREATE TABLE bitrix_sync.webhook_inbox (
|
||||
id uuid PRIMARY KEY,
|
||||
receiver_type varchar(16) NOT NULL CHECK (receiver_type IN ('contact','alert')),
|
||||
event_type varchar(64) NOT NULL,
|
||||
event_id varchar(255),
|
||||
source_timestamp timestamptz,
|
||||
external_entity_id varchar(128) NOT NULL,
|
||||
dedup_fingerprint varchar(64),
|
||||
source_ip inet,
|
||||
status varchar(16) NOT NULL CHECK (status IN
|
||||
('received','coalesced','processing','processed','retry_wait','dead_letter')),
|
||||
coalesced_count integer NOT NULL DEFAULT 1,
|
||||
attempt_count integer NOT NULL DEFAULT 0,
|
||||
next_attempt_at timestamptz NOT NULL DEFAULT now(),
|
||||
locked_by varchar(128),
|
||||
locked_until timestamptz,
|
||||
lease_token uuid,
|
||||
received_at timestamptz NOT NULL,
|
||||
last_received_at timestamptz NOT NULL,
|
||||
processed_at timestamptz,
|
||||
safe_error_code varchar(64)
|
||||
);
|
||||
CREATE UNIQUE INDEX uq_webhook_event_id
|
||||
ON bitrix_sync.webhook_inbox(receiver_type,event_id) WHERE event_id IS NOT NULL;
|
||||
CREATE INDEX ix_webhook_claim
|
||||
ON bitrix_sync.webhook_inbox(status,next_attempt_at,received_at);
|
||||
CREATE UNIQUE INDEX uq_webhook_active_entity
|
||||
ON bitrix_sync.webhook_inbox(receiver_type,external_entity_id)
|
||||
WHERE status IN ('received','processing','retry_wait');
|
||||
"""
|
||||
)
|
||||
_execute_script(
|
||||
"""
|
||||
CREATE SEQUENCE bitrix_sync.business_alert_number_seq;
|
||||
CREATE TABLE bitrix_sync.business_alerts (
|
||||
id uuid PRIMARY KEY,
|
||||
alert_number bigint NOT NULL DEFAULT nextval('bitrix_sync.business_alert_number_seq'),
|
||||
fingerprint varchar(64) NOT NULL,
|
||||
alert_type varchar(64) NOT NULL,
|
||||
severity varchar(16) NOT NULL CHECK (severity IN ('info','warning','critical')),
|
||||
app_user_id uuid,
|
||||
current_external_id varchar(128),
|
||||
selected_external_id varchar(128),
|
||||
candidate_external_ids text[] NOT NULL DEFAULT '{}',
|
||||
remote_item_id varchar(128),
|
||||
remote_stage_id varchar(128),
|
||||
previous_alert_id uuid REFERENCES bitrix_sync.business_alerts(id),
|
||||
workflow_id uuid REFERENCES bitrix_sync.workflow_instances(id),
|
||||
status varchar(24) NOT NULL CHECK (status IN
|
||||
('open','in_progress','resolved','closed_without_resolution','remote_missing')),
|
||||
occurrence_count integer NOT NULL DEFAULT 1,
|
||||
first_occurred_at timestamptz NOT NULL,
|
||||
last_occurred_at timestamptz NOT NULL,
|
||||
created_at timestamptz NOT NULL DEFAULT now(),
|
||||
updated_at timestamptz NOT NULL DEFAULT now()
|
||||
);
|
||||
CREATE UNIQUE INDEX uq_business_alert_open
|
||||
ON bitrix_sync.business_alerts(alert_type,fingerprint) WHERE status='open';
|
||||
CREATE INDEX ix_business_alert_remote
|
||||
ON bitrix_sync.business_alerts(remote_item_id) WHERE remote_item_id IS NOT NULL;
|
||||
|
||||
CREATE TABLE bitrix_sync.rebind_requests (
|
||||
id uuid PRIMARY KEY,
|
||||
user_id uuid NOT NULL,
|
||||
old_external_id varchar(128),
|
||||
target_external_id varchar(128) NOT NULL,
|
||||
reason varchar(500) NOT NULL,
|
||||
operator_id varchar(128) NOT NULL,
|
||||
workflow_id uuid NOT NULL UNIQUE REFERENCES bitrix_sync.workflow_instances(id),
|
||||
status varchar(24) NOT NULL CHECK (status IN
|
||||
('pending','processing','retry_wait','succeeded','failed','cancelled')),
|
||||
safe_error_code varchar(64),
|
||||
requested_at timestamptz NOT NULL DEFAULT now(),
|
||||
completed_at timestamptz,
|
||||
created_at timestamptz NOT NULL DEFAULT now(),
|
||||
updated_at timestamptz NOT NULL DEFAULT now()
|
||||
);
|
||||
CREATE INDEX ix_rebind_claim
|
||||
ON bitrix_sync.rebind_requests(status,requested_at)
|
||||
WHERE status IN ('pending','retry_wait');
|
||||
|
||||
CREATE TABLE bitrix_sync.contact_snapshots (
|
||||
id uuid PRIMARY KEY,
|
||||
mapping_id uuid NOT NULL REFERENCES bitrix_sync.entity_external_mapping(id),
|
||||
user_id uuid NOT NULL,
|
||||
external_id varchar(128) NOT NULL,
|
||||
full_name_hash varchar(64),
|
||||
email_hash varchar(64),
|
||||
phone_hash varchar(64),
|
||||
citizenship_hash varchar(64),
|
||||
citizenship_enum_id varchar(128),
|
||||
citizenship_dictionary_loaded_at timestamptz,
|
||||
source_updated_at timestamptz,
|
||||
app_version varchar(128),
|
||||
last_applied_source varchar(24) NOT NULL CHECK (last_applied_source IN
|
||||
('webhook','reconciliation','app_create','app_update')),
|
||||
last_webhook_received_at timestamptz,
|
||||
last_webhook_source_at timestamptz,
|
||||
profile_stale boolean NOT NULL DEFAULT false,
|
||||
created_at timestamptz NOT NULL DEFAULT now(),
|
||||
updated_at timestamptz NOT NULL DEFAULT now(),
|
||||
UNIQUE(mapping_id)
|
||||
);
|
||||
CREATE INDEX ix_contact_snapshot_source
|
||||
ON bitrix_sync.contact_snapshots(source_updated_at);
|
||||
|
||||
CREATE TABLE bitrix_sync.settings_versions (
|
||||
id uuid PRIMARY KEY,
|
||||
version bigint NOT NULL UNIQUE,
|
||||
validation_status varchar(16) NOT NULL
|
||||
CHECK (validation_status IN ('pending','valid','invalid')),
|
||||
validation_errors jsonb NOT NULL DEFAULT '[]',
|
||||
active boolean NOT NULL DEFAULT false,
|
||||
created_by varchar(128) NOT NULL,
|
||||
created_at timestamptz NOT NULL DEFAULT now(),
|
||||
activated_at timestamptz
|
||||
);
|
||||
CREATE UNIQUE INDEX uq_settings_version_active
|
||||
ON bitrix_sync.settings_versions(active) WHERE active=true;
|
||||
|
||||
CREATE TABLE bitrix_sync.settings (
|
||||
id uuid PRIMARY KEY,
|
||||
version_id uuid NOT NULL REFERENCES bitrix_sync.settings_versions(id),
|
||||
key varchar(128) NOT NULL,
|
||||
value_type varchar(16) NOT NULL
|
||||
CHECK (value_type IN ('integer','number','boolean','object')),
|
||||
value_json jsonb NOT NULL,
|
||||
validation_status varchar(16) NOT NULL CHECK (validation_status IN ('valid','invalid')),
|
||||
active boolean NOT NULL DEFAULT false,
|
||||
created_at timestamptz NOT NULL DEFAULT now(),
|
||||
updated_at timestamptz NOT NULL DEFAULT now(),
|
||||
UNIQUE(version_id,key)
|
||||
);
|
||||
CREATE INDEX ix_settings_active ON bitrix_sync.settings(key) WHERE active=true;
|
||||
|
||||
CREATE TABLE bitrix_sync.technical_dead_letters (
|
||||
id uuid PRIMARY KEY,
|
||||
workflow_id uuid REFERENCES bitrix_sync.workflow_instances(id),
|
||||
command_id uuid REFERENCES bitrix_sync.crm_commands(id),
|
||||
operation varchar(64) NOT NULL,
|
||||
safe_error_code varchar(64) NOT NULL,
|
||||
attempt_count integer NOT NULL DEFAULT 0,
|
||||
deadline_at timestamptz,
|
||||
correlation_id uuid,
|
||||
failed_at timestamptz NOT NULL,
|
||||
created_at timestamptz NOT NULL DEFAULT now()
|
||||
);
|
||||
CREATE INDEX ix_technical_dlq_failed
|
||||
ON bitrix_sync.technical_dead_letters(failed_at DESC);
|
||||
|
||||
CREATE TABLE bitrix_sync.reconciliation_cursors (
|
||||
job_type varchar(64) PRIMARY KEY,
|
||||
watermark timestamptz NOT NULL,
|
||||
overlap_seconds integer NOT NULL CHECK (overlap_seconds>=0),
|
||||
last_success_at timestamptz,
|
||||
last_scanned_count integer NOT NULL DEFAULT 0,
|
||||
last_updated_count integer NOT NULL DEFAULT 0,
|
||||
recovered_without_webhook_count integer NOT NULL DEFAULT 0,
|
||||
created_at timestamptz NOT NULL DEFAULT now(),
|
||||
updated_at timestamptz NOT NULL DEFAULT now()
|
||||
);
|
||||
|
||||
CREATE TABLE bitrix_sync.limiter_coordination (
|
||||
limiter_key varchar(128) PRIMARY KEY,
|
||||
tokens numeric(12,6) NOT NULL,
|
||||
capacity numeric(12,6) NOT NULL CHECK (capacity>0),
|
||||
refill_per_second numeric(12,6) NOT NULL CHECK (refill_per_second>0),
|
||||
updated_at timestamptz NOT NULL,
|
||||
blocked_until timestamptz,
|
||||
method_class_blocks jsonb NOT NULL DEFAULT '{}',
|
||||
fencing_token bigint NOT NULL DEFAULT 0
|
||||
);
|
||||
|
||||
CREATE TABLE bitrix_sync.citizenship_dictionary (
|
||||
enum_id varchar(128) PRIMARY KEY,
|
||||
display_value varchar(255) NOT NULL,
|
||||
loaded_at timestamptz NOT NULL,
|
||||
expires_at timestamptz NOT NULL
|
||||
);
|
||||
"""
|
||||
)
|
||||
op.execute(
|
||||
"""
|
||||
CREATE OR REPLACE FUNCTION bitrix_sync.request_bitrix_contact_rebind(
|
||||
p_user_id uuid,
|
||||
p_target_b24_id varchar,
|
||||
p_reason varchar,
|
||||
p_operator_id varchar
|
||||
) RETURNS uuid
|
||||
LANGUAGE plpgsql
|
||||
SECURITY DEFINER
|
||||
SET search_path=bitrix_sync,pg_temp
|
||||
AS $$
|
||||
DECLARE
|
||||
v_old_id varchar(128);
|
||||
v_request_id uuid := gen_random_uuid();
|
||||
v_workflow_id uuid := gen_random_uuid();
|
||||
BEGIN
|
||||
IF p_target_b24_id !~ '^[1-9][0-9]*$'
|
||||
OR length(trim(p_reason)) < 5
|
||||
OR length(trim(p_operator_id)) < 1 THEN
|
||||
RAISE EXCEPTION 'invalid rebind request' USING ERRCODE='22023';
|
||||
END IF;
|
||||
SELECT external_id INTO v_old_id
|
||||
FROM bitrix_sync.entity_external_mapping
|
||||
WHERE entity_id=p_user_id AND entity_type='contact'
|
||||
AND external_system='bitrix24' AND status='active'
|
||||
FOR UPDATE;
|
||||
IF EXISTS (
|
||||
SELECT 1 FROM bitrix_sync.entity_external_mapping
|
||||
WHERE external_system='bitrix24' AND external_entity_type='contact'
|
||||
AND external_id=p_target_b24_id AND status='active'
|
||||
AND entity_id<>p_user_id
|
||||
) THEN
|
||||
RAISE EXCEPTION 'target contact has another active mapping'
|
||||
USING ERRCODE='23505';
|
||||
END IF;
|
||||
INSERT INTO bitrix_sync.workflow_instances
|
||||
(id,workflow_type,user_id,external_id,state,current_step,deadline_at,created_at,updated_at)
|
||||
VALUES
|
||||
(v_workflow_id,'contact.rebind',p_user_id,p_target_b24_id,'created',
|
||||
'validate_contacts',now()+interval '24 hours',now(),now());
|
||||
INSERT INTO bitrix_sync.rebind_requests
|
||||
(id,user_id,old_external_id,target_external_id,reason,operator_id,
|
||||
workflow_id,status,requested_at,created_at,updated_at)
|
||||
VALUES
|
||||
(v_request_id,p_user_id,v_old_id,p_target_b24_id,trim(p_reason),
|
||||
trim(p_operator_id),v_workflow_id,'pending',now(),now(),now());
|
||||
RETURN v_request_id;
|
||||
END;
|
||||
$$;
|
||||
"""
|
||||
)
|
||||
op.execute(
|
||||
"""
|
||||
REVOKE ALL ON FUNCTION
|
||||
bitrix_sync.request_bitrix_contact_rebind(uuid,varchar,varchar,varchar)
|
||||
FROM PUBLIC
|
||||
"""
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
raise RuntimeError("bitrix_sync production migration is forward-only")
|
||||
+85
@@ -0,0 +1,85 @@
|
||||
"""Adopt the App queue contract and migrate legacy mappings.
|
||||
|
||||
Revision ID: 0002_app_queue_contract
|
||||
Revises: 0001_bitrix_sync_full
|
||||
Create Date: 2026-08-06
|
||||
"""
|
||||
|
||||
from collections.abc import Sequence
|
||||
|
||||
from alembic import op
|
||||
|
||||
revision: str = "0002_app_queue_contract"
|
||||
down_revision: str | None = "0001_bitrix_sync_full"
|
||||
branch_labels: str | Sequence[str] | None = None
|
||||
depends_on: str | Sequence[str] | None = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.execute(
|
||||
"""
|
||||
DO $$
|
||||
BEGIN
|
||||
IF to_regclass('han_app.entity_external_mapping') IS NOT NULL THEN
|
||||
EXECUTE $copy$
|
||||
INSERT INTO bitrix_sync.entity_external_mapping
|
||||
(id,entity_type,entity_id,external_system,external_entity_type,
|
||||
external_id,status,opened_at,closed_at,close_reason,created_at,updated_at)
|
||||
SELECT id,entity_type,entity_id,'bitrix24','contact',
|
||||
external_id,'active',coalesce(created_at,now()),NULL,NULL,
|
||||
coalesce(created_at,now()),coalesce(created_at,now())
|
||||
FROM han_app.entity_external_mapping
|
||||
ON CONFLICT DO NOTHING
|
||||
$copy$;
|
||||
IF EXISTS (
|
||||
SELECT 1 FROM han_app.entity_external_mapping old
|
||||
LEFT JOIN bitrix_sync.entity_external_mapping new ON new.id=old.id
|
||||
WHERE new.id IS NULL
|
||||
) THEN
|
||||
RAISE EXCEPTION 'legacy mapping migration verification failed';
|
||||
END IF;
|
||||
END IF;
|
||||
END $$;
|
||||
"""
|
||||
)
|
||||
|
||||
op.execute(
|
||||
"""
|
||||
INSERT INTO bitrix_sync.settings_versions
|
||||
(id,version,validation_status,active,created_by,created_at,activated_at)
|
||||
VALUES ('00000000-0000-0000-0000-000000000001',1,'valid',true,'migration',now(),now())
|
||||
ON CONFLICT DO NOTHING
|
||||
"""
|
||||
)
|
||||
op.execute(
|
||||
"""
|
||||
INSERT INTO bitrix_sync.settings
|
||||
(id,version_id,key,value_type,value_json,validation_status,active,created_at,updated_at)
|
||||
VALUES
|
||||
(gen_random_uuid(),'00000000-0000-0000-0000-000000000001','worker','object',
|
||||
jsonb_build_object(
|
||||
'batch_size',20,'batch_wait_ms',200,'claim_size',20,'lease_seconds',60,
|
||||
'limiter_refill_per_sec',2,'limiter_burst',2,'max_in_flight',2,
|
||||
'retry_base_seconds',1,'retry_max_seconds',900,'retry_horizon_seconds',86400
|
||||
),
|
||||
'valid',true,now(),now()),
|
||||
(gen_random_uuid(),'00000000-0000-0000-0000-000000000001','reconciliation','object',
|
||||
jsonb_build_object(
|
||||
'contact_interval_seconds',900,'alert_interval_seconds',3600,
|
||||
'overlap_seconds',300,'recovered_spike_threshold',20
|
||||
),
|
||||
'valid',true,now(),now()),
|
||||
(gen_random_uuid(),'00000000-0000-0000-0000-000000000001','business_alerts','object',
|
||||
jsonb_build_object(
|
||||
'entity_type_id',NULL,'category_id',NULL,'stage_new',NULL,
|
||||
'stage_in_progress',NULL,'stage_resolved',NULL,
|
||||
'stage_closed_without_resolution',NULL,'sla_business_hours',8
|
||||
),
|
||||
'valid',true,now(),now())
|
||||
ON CONFLICT DO NOTHING;
|
||||
"""
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
raise RuntimeError("App queue contract migration is forward-only")
|
||||
@@ -0,0 +1 @@
|
||||
"""HAN Bitrix24 synchronization service."""
|
||||
@@ -0,0 +1,137 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import ipaddress
|
||||
import re
|
||||
from functools import cached_property
|
||||
from pathlib import Path
|
||||
from urllib.parse import urlsplit
|
||||
|
||||
from pydantic import Field, SecretStr, model_validator
|
||||
from pydantic_settings import BaseSettings, SettingsConfigDict
|
||||
|
||||
FIELD_RE = re.compile(r"^UF_CRM_[0-9]+$")
|
||||
MEMBER_RE = re.compile(r"^[A-Za-z0-9_-]{8,128}$")
|
||||
|
||||
|
||||
def _read_secret(value: SecretStr | None, path: Path | None) -> SecretStr | None:
|
||||
if value and value.get_secret_value():
|
||||
return value
|
||||
if path:
|
||||
text = path.read_text(encoding="utf-8").strip()
|
||||
return SecretStr(text) if text else None
|
||||
return None
|
||||
|
||||
|
||||
class Settings(BaseSettings):
|
||||
model_config = SettingsConfigDict(env_prefix="BITRIX_SYNC_", extra="ignore")
|
||||
|
||||
enabled: bool = False
|
||||
mode: str = "disabled"
|
||||
database_url: SecretStr | None = None
|
||||
database_url_file: Path | None = None
|
||||
crm_rest_webhook_url: SecretStr | None = None
|
||||
crm_rest_webhook_url_file: Path | None = None
|
||||
contact_receiver_token: SecretStr | None = None
|
||||
contact_receiver_token_file: Path | None = None
|
||||
contact_receiver_previous_token: SecretStr | None = None
|
||||
alert_receiver_token: SecretStr | None = None
|
||||
alert_receiver_token_file: Path | None = None
|
||||
alert_receiver_previous_token: SecretStr | None = None
|
||||
service_token: SecretStr | None = None
|
||||
service_token_file: Path | None = None
|
||||
|
||||
portal_host: str | None = None
|
||||
portal_member_id: str | None = None
|
||||
public_base_url: str | None = None
|
||||
contact_user_id_field: str | None = None
|
||||
contact_registered_field: str | None = None
|
||||
contact_citizenship_field: str | None = None
|
||||
webhook_allowed_cidrs: str = ""
|
||||
|
||||
http_timeout_sec: float = Field(default=10, ge=1, le=60)
|
||||
db_pool_size: int = Field(default=5, ge=1, le=30)
|
||||
webhook_max_body_bytes: int = Field(default=16_384, ge=1024, le=65_536)
|
||||
webhook_max_fields: int = Field(default=24, ge=8, le=64)
|
||||
lease_seconds: int = Field(default=60, ge=10, le=600)
|
||||
claim_size: int = Field(default=20, ge=1, le=100)
|
||||
batch_size: int = Field(default=20, ge=1, le=50)
|
||||
batch_wait_ms: int = Field(default=200, ge=10, le=5000)
|
||||
limiter_refill_per_sec: float = Field(default=2, gt=0, le=50)
|
||||
limiter_burst: int = Field(default=2, ge=1, le=50)
|
||||
max_in_flight: int = Field(default=2, ge=1, le=20)
|
||||
retry_base_seconds: float = Field(default=1, ge=0.1, le=60)
|
||||
retry_max_seconds: float = Field(default=900, ge=1, le=3600)
|
||||
retry_horizon_seconds: int = Field(default=86_400, ge=60, le=604_800)
|
||||
reconciliation_overlap_seconds: int = Field(default=300, ge=0, le=3600)
|
||||
reconciliation_interval_seconds: int = Field(default=900, ge=60, le=86_400)
|
||||
|
||||
@model_validator(mode="after")
|
||||
def validate_mode(self) -> Settings:
|
||||
self.database_url = _read_secret(self.database_url, self.database_url_file)
|
||||
self.crm_rest_webhook_url = _read_secret(
|
||||
self.crm_rest_webhook_url, self.crm_rest_webhook_url_file
|
||||
)
|
||||
self.contact_receiver_token = _read_secret(
|
||||
self.contact_receiver_token, self.contact_receiver_token_file
|
||||
)
|
||||
self.alert_receiver_token = _read_secret(
|
||||
self.alert_receiver_token, self.alert_receiver_token_file
|
||||
)
|
||||
self.service_token = _read_secret(self.service_token, self.service_token_file)
|
||||
|
||||
expected_mode = "full" if self.enabled else "disabled"
|
||||
if self.mode != expected_mode:
|
||||
raise ValueError(f"mode must be {expected_mode!r} when enabled={self.enabled}")
|
||||
if not self.enabled:
|
||||
return self
|
||||
|
||||
required = {
|
||||
"database_url": self.database_url,
|
||||
"crm_rest_webhook_url": self.crm_rest_webhook_url,
|
||||
"contact_receiver_token": self.contact_receiver_token,
|
||||
"alert_receiver_token": self.alert_receiver_token,
|
||||
"service_token": self.service_token,
|
||||
"portal_host": self.portal_host,
|
||||
"portal_member_id": self.portal_member_id,
|
||||
"public_base_url": self.public_base_url,
|
||||
"contact_user_id_field": self.contact_user_id_field,
|
||||
"contact_registered_field": self.contact_registered_field,
|
||||
"contact_citizenship_field": self.contact_citizenship_field,
|
||||
}
|
||||
missing = [name for name, value in required.items() if not value]
|
||||
if missing:
|
||||
raise ValueError("missing full-mode settings: " + ", ".join(missing))
|
||||
for name in (
|
||||
"contact_user_id_field",
|
||||
"contact_registered_field",
|
||||
"contact_citizenship_field",
|
||||
):
|
||||
if not FIELD_RE.fullmatch(str(getattr(self, name))):
|
||||
raise ValueError(f"{name} must match UF_CRM_<digits>")
|
||||
if not MEMBER_RE.fullmatch(str(self.portal_member_id)):
|
||||
raise ValueError("portal_member_id has invalid format")
|
||||
|
||||
crm = urlsplit(self.crm_rest_webhook_url.get_secret_value())
|
||||
public = urlsplit(str(self.public_base_url))
|
||||
if crm.scheme != "https" or crm.hostname != self.portal_host or crm.port not in (None, 443):
|
||||
raise ValueError("CRM URL must be HTTPS on the approved portal host")
|
||||
if public.scheme != "https" or not public.hostname or public.query or public.fragment:
|
||||
raise ValueError("public_base_url must be a query-free HTTPS origin")
|
||||
if not self.allowed_networks:
|
||||
raise ValueError("webhook_allowed_cidrs cannot be empty in full mode")
|
||||
return self
|
||||
|
||||
@cached_property
|
||||
def allowed_networks(self) -> tuple[ipaddress.IPv4Network | ipaddress.IPv6Network, ...]:
|
||||
values = [item.strip() for item in self.webhook_allowed_cidrs.split(",") if item.strip()]
|
||||
return tuple(ipaddress.ip_network(item, strict=True) for item in values)
|
||||
|
||||
@staticmethod
|
||||
def rest_field_name(field: str) -> str:
|
||||
if not FIELD_RE.fullmatch(field):
|
||||
raise ValueError("invalid Bitrix custom field")
|
||||
return "ufCrm_" + field.removeprefix("UF_CRM_")
|
||||
|
||||
|
||||
def load_settings() -> Settings:
|
||||
return Settings()
|
||||
@@ -0,0 +1,185 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import ssl
|
||||
from dataclasses import dataclass
|
||||
from datetime import UTC, datetime
|
||||
from enum import StrEnum
|
||||
from typing import Any
|
||||
from urllib.parse import urljoin, urlsplit
|
||||
|
||||
import httpx
|
||||
|
||||
|
||||
class CrmOutcome(StrEnum):
|
||||
SUCCEEDED = "succeeded"
|
||||
RETRY = "retry"
|
||||
UNCERTAIN = "uncertain"
|
||||
PERMANENT = "permanent"
|
||||
RATE_LIMITED = "rate_limited"
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class CrmResult:
|
||||
outcome: CrmOutcome
|
||||
result: Any = None
|
||||
error_code: str | None = None
|
||||
http_status: int | None = None
|
||||
retry_after: float | None = None
|
||||
|
||||
|
||||
class CrmClient:
|
||||
"""Host-locked, verified-TLS Bitrix client; redirects are never followed."""
|
||||
|
||||
def __init__(self, credential_url: str, approved_host: str, timeout: float) -> None:
|
||||
parsed = urlsplit(credential_url)
|
||||
if parsed.scheme != "https" or parsed.hostname != approved_host:
|
||||
raise ValueError("credential URL is outside approved Bitrix host")
|
||||
self._base_url = credential_url.rstrip("/") + "/"
|
||||
self._host = approved_host
|
||||
self._client = httpx.AsyncClient(
|
||||
timeout=httpx.Timeout(timeout),
|
||||
verify=ssl.create_default_context(),
|
||||
follow_redirects=False,
|
||||
limits=httpx.Limits(max_connections=4, max_keepalive_connections=2),
|
||||
headers={"Accept": "application/json"},
|
||||
)
|
||||
|
||||
async def close(self) -> None:
|
||||
await self._client.aclose()
|
||||
|
||||
async def call(self, method: str, params: dict[str, Any], *, mutating: bool) -> CrmResult:
|
||||
if method not in ALLOWED_METHODS:
|
||||
raise ValueError("unapproved CRM method")
|
||||
url = urljoin(self._base_url, method + ".json")
|
||||
if urlsplit(url).hostname != self._host:
|
||||
raise ValueError("CRM host changed during URL construction")
|
||||
try:
|
||||
response = await self._client.post(url, json=params)
|
||||
except (httpx.ConnectError, httpx.ReadError, httpx.RemoteProtocolError):
|
||||
return CrmResult(CrmOutcome.RETRY, error_code="crm_network")
|
||||
except httpx.TimeoutException:
|
||||
outcome = CrmOutcome.UNCERTAIN if mutating else CrmOutcome.RETRY
|
||||
return CrmResult(outcome, error_code="crm_timeout")
|
||||
if response.is_redirect:
|
||||
return CrmResult(
|
||||
CrmOutcome.PERMANENT,
|
||||
error_code="crm_redirect_rejected",
|
||||
http_status=response.status_code,
|
||||
)
|
||||
retry_after = _retry_after(response)
|
||||
if response.status_code in (408, 429) or response.status_code >= 500:
|
||||
return CrmResult(
|
||||
CrmOutcome.RETRY,
|
||||
error_code=f"crm_http_{response.status_code}",
|
||||
http_status=response.status_code,
|
||||
retry_after=retry_after,
|
||||
)
|
||||
try:
|
||||
payload = response.json()
|
||||
except ValueError:
|
||||
return CrmResult(
|
||||
CrmOutcome.PERMANENT,
|
||||
error_code="crm_malformed_response",
|
||||
http_status=response.status_code,
|
||||
)
|
||||
error = payload.get("error")
|
||||
if error == "QUERY_LIMIT_EXCEEDED":
|
||||
return CrmResult(CrmOutcome.RATE_LIMITED, error_code=error, retry_after=retry_after)
|
||||
if error == "OPERATION_TIME_LIMIT":
|
||||
return CrmResult(CrmOutcome.RETRY, error_code=error, retry_after=retry_after)
|
||||
if error:
|
||||
permanent = error in {
|
||||
"ERROR_METHOD_NOT_FOUND",
|
||||
"ERROR_WRONG_AUTH_TYPE",
|
||||
"INVALID_CREDENTIALS",
|
||||
"ACCESS_DENIED",
|
||||
"ERROR_ARGUMENT",
|
||||
}
|
||||
return CrmResult(
|
||||
CrmOutcome.PERMANENT if permanent else CrmOutcome.RETRY,
|
||||
error_code=str(error)[:64],
|
||||
http_status=response.status_code,
|
||||
)
|
||||
return CrmResult(CrmOutcome.SUCCEEDED, result=payload.get("result"))
|
||||
|
||||
async def batch(self, commands: list[tuple[str, dict[str, Any]]]) -> list[CrmResult]:
|
||||
if not commands:
|
||||
return []
|
||||
if len(commands) > 50:
|
||||
raise ValueError("Bitrix batch limit exceeded")
|
||||
cmd = {
|
||||
str(index): f"{method}?{httpx.QueryParams(params)}"
|
||||
for index, (method, params) in enumerate(commands)
|
||||
if method in ALLOWED_METHODS
|
||||
}
|
||||
batch = await self.call("batch", {"halt": 0, "cmd": cmd}, mutating=True)
|
||||
if batch.outcome != CrmOutcome.SUCCEEDED:
|
||||
return [batch for _ in commands]
|
||||
result = batch.result or {}
|
||||
successes = result.get("result", {})
|
||||
errors = result.get("result_error", {})
|
||||
return [
|
||||
CrmResult(CrmOutcome.SUCCEEDED, result=successes.get(str(i)))
|
||||
if str(i) in successes
|
||||
else CrmResult(CrmOutcome.RETRY, error_code=str(errors.get(str(i), "batch_missing")))
|
||||
for i in range(len(commands))
|
||||
]
|
||||
|
||||
|
||||
ALLOWED_METHODS = frozenset(
|
||||
{
|
||||
"batch",
|
||||
"crm.duplicate.findbycomm",
|
||||
"crm.contact.get",
|
||||
"crm.contact.add",
|
||||
"crm.contact.update",
|
||||
"crm.contact.userfield.list",
|
||||
"crm.item.list",
|
||||
"crm.item.get",
|
||||
"crm.item.add",
|
||||
"crm.item.update",
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def _retry_after(response: httpx.Response) -> float | None:
|
||||
value = response.headers.get("Retry-After")
|
||||
if not value:
|
||||
return None
|
||||
try:
|
||||
return max(0.0, float(value))
|
||||
except ValueError:
|
||||
try:
|
||||
retry_at = datetime.fromisoformat(value).astimezone(UTC)
|
||||
return max(0.0, (retry_at - datetime.now(UTC)).total_seconds())
|
||||
except ValueError:
|
||||
return None
|
||||
|
||||
|
||||
class TokenBucket:
|
||||
def __init__(self, refill_per_second: float, burst: int, max_in_flight: int) -> None:
|
||||
self.refill_per_second = refill_per_second
|
||||
self.burst = float(burst)
|
||||
self.tokens = float(burst)
|
||||
self.updated_at = asyncio.get_running_loop().time()
|
||||
self._lock = asyncio.Lock()
|
||||
self._slots = asyncio.Semaphore(max_in_flight)
|
||||
|
||||
async def acquire(self) -> None:
|
||||
await self._slots.acquire()
|
||||
while True:
|
||||
async with self._lock:
|
||||
now = asyncio.get_running_loop().time()
|
||||
self.tokens = min(
|
||||
self.burst, self.tokens + (now - self.updated_at) * self.refill_per_second
|
||||
)
|
||||
self.updated_at = now
|
||||
if self.tokens >= 1:
|
||||
self.tokens -= 1
|
||||
return
|
||||
delay = (1 - self.tokens) / self.refill_per_second
|
||||
await asyncio.sleep(delay)
|
||||
|
||||
def release(self) -> None:
|
||||
self._slots.release()
|
||||
@@ -0,0 +1,106 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import random
|
||||
import re
|
||||
from dataclasses import dataclass
|
||||
from datetime import UTC, datetime
|
||||
from enum import StrEnum
|
||||
from typing import Any
|
||||
|
||||
PHONE_RE = re.compile(r"^\+7[0-9]{10}$")
|
||||
EMAIL_RE = re.compile(r"^[^@\s]+@[^@\s]+\.[^@\s]+$")
|
||||
|
||||
|
||||
class WorkflowState(StrEnum):
|
||||
CREATED = "created"
|
||||
RUNNING = "running"
|
||||
WAITING_CRM = "waiting_crm"
|
||||
WAITING_RETRY = "waiting_retry"
|
||||
WAITING_MANUAL = "waiting_manual"
|
||||
SUCCEEDED = "succeeded"
|
||||
FAILED = "failed"
|
||||
CANCELLED = "cancelled"
|
||||
|
||||
|
||||
TRANSITIONS: dict[WorkflowState, frozenset[WorkflowState]] = {
|
||||
WorkflowState.CREATED: frozenset({WorkflowState.RUNNING, WorkflowState.CANCELLED}),
|
||||
WorkflowState.RUNNING: frozenset(
|
||||
{
|
||||
WorkflowState.WAITING_CRM,
|
||||
WorkflowState.WAITING_RETRY,
|
||||
WorkflowState.WAITING_MANUAL,
|
||||
WorkflowState.SUCCEEDED,
|
||||
WorkflowState.FAILED,
|
||||
WorkflowState.CANCELLED,
|
||||
}
|
||||
),
|
||||
WorkflowState.WAITING_CRM: frozenset(
|
||||
{
|
||||
WorkflowState.RUNNING,
|
||||
WorkflowState.WAITING_RETRY,
|
||||
WorkflowState.WAITING_MANUAL,
|
||||
WorkflowState.FAILED,
|
||||
}
|
||||
),
|
||||
WorkflowState.WAITING_RETRY: frozenset(
|
||||
{WorkflowState.RUNNING, WorkflowState.WAITING_MANUAL, WorkflowState.FAILED}
|
||||
),
|
||||
WorkflowState.WAITING_MANUAL: frozenset(
|
||||
{WorkflowState.RUNNING, WorkflowState.CANCELLED}
|
||||
),
|
||||
WorkflowState.SUCCEEDED: frozenset(),
|
||||
WorkflowState.FAILED: frozenset(),
|
||||
WorkflowState.CANCELLED: frozenset(),
|
||||
}
|
||||
|
||||
|
||||
def assert_transition(current: WorkflowState, target: WorkflowState) -> None:
|
||||
if target not in TRANSITIONS[current]:
|
||||
raise ValueError(f"forbidden workflow transition {current} -> {target}")
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ContactCandidate:
|
||||
b24_id: str
|
||||
created_at: datetime
|
||||
crm_user_id: str | None
|
||||
|
||||
|
||||
def validate_phone(value: str) -> str:
|
||||
if not PHONE_RE.fullmatch(value):
|
||||
raise ValueError("phone must be Russian E.164 +7XXXXXXXXXX")
|
||||
return value
|
||||
|
||||
|
||||
def choose_newest(candidates: list[ContactCandidate]) -> ContactCandidate | None:
|
||||
if not candidates:
|
||||
return None
|
||||
return max(candidates, key=lambda item: (item.created_at, int(item.b24_id)))
|
||||
|
||||
|
||||
def select_email(items: list[dict[str, Any]]) -> str | None:
|
||||
valid = [
|
||||
item
|
||||
for item in items
|
||||
if isinstance(item.get("VALUE"), str) and EMAIL_RE.fullmatch(item["VALUE"])
|
||||
]
|
||||
work = next((item for item in valid if item.get("VALUE_TYPE") == "WORK"), None)
|
||||
selected = work or (valid[0] if valid else None)
|
||||
return selected["VALUE"] if selected else None
|
||||
|
||||
|
||||
def safe_hash(value: str | None) -> str | None:
|
||||
return hashlib.sha256(value.encode()).hexdigest() if value is not None else None
|
||||
|
||||
|
||||
def full_jitter_delay(
|
||||
attempt: int, base: float, maximum: float, *, rng: random.Random | None = None
|
||||
) -> float:
|
||||
ceiling = min(maximum, base * (2 ** max(0, attempt - 1)))
|
||||
return (rng or random.SystemRandom()).uniform(0, ceiling)
|
||||
|
||||
|
||||
def parse_crm_datetime(value: str) -> datetime:
|
||||
parsed = datetime.fromisoformat(value.replace("Z", "+00:00"))
|
||||
return parsed.astimezone(UTC)
|
||||
@@ -0,0 +1,629 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import uuid
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import text
|
||||
|
||||
from app.config import Settings
|
||||
from app.crm import CrmClient, CrmOutcome, CrmResult
|
||||
from app.domain import (
|
||||
ContactCandidate,
|
||||
choose_newest,
|
||||
full_jitter_delay,
|
||||
parse_crm_datetime,
|
||||
select_email,
|
||||
validate_phone,
|
||||
)
|
||||
from app.repository import LeasedTask, LeasedWebhook, Profile, Repository
|
||||
|
||||
|
||||
class BusinessConflict(Exception):
|
||||
def __init__(self, code: str, candidates: list[str] | None = None) -> None:
|
||||
self.code = code
|
||||
self.candidates = candidates or []
|
||||
super().__init__(code)
|
||||
|
||||
|
||||
@dataclass
|
||||
class WorkflowEngine:
|
||||
repository: Repository
|
||||
crm: CrmClient
|
||||
settings: Settings
|
||||
_in_flight: asyncio.Semaphore = field(init=False, repr=False)
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
self._in_flight = asyncio.Semaphore(self.settings.max_in_flight)
|
||||
|
||||
async def process(self, task: LeasedTask) -> None:
|
||||
workflow_id = await self.repository.create_workflow(task)
|
||||
profile = await self.repository.load_profile(task.user_id)
|
||||
if profile is None:
|
||||
await self._manual(workflow_id, "profile_missing", task.user_id)
|
||||
await self.repository.complete_task(task, workflow_id)
|
||||
return
|
||||
try:
|
||||
if task.task_type == "contact.map_or_create":
|
||||
await self._map_or_create(workflow_id, profile)
|
||||
elif task.task_type == "contact.update":
|
||||
await self._update(workflow_id, profile)
|
||||
elif task.task_type == "contact.deactivate":
|
||||
await self._deactivate(workflow_id, profile)
|
||||
else:
|
||||
await self._technical_failure(workflow_id, "unknown_task_type")
|
||||
await self.repository.complete_task(task, workflow_id)
|
||||
except BusinessConflict as exc:
|
||||
await self._alert(workflow_id, task.user_id, exc.code, exc.candidates)
|
||||
await self._manual(workflow_id, exc.code, task.user_id)
|
||||
await self.repository.complete_task(task, workflow_id)
|
||||
except RetryableWorkflow as exc:
|
||||
delay = exc.retry_after or full_jitter_delay(
|
||||
task.attempt_count + 1,
|
||||
self.settings.retry_base_seconds,
|
||||
self.settings.retry_max_seconds,
|
||||
)
|
||||
await self.repository.retry_task(task, exc.code, delay)
|
||||
|
||||
async def process_webhook(self, item: LeasedWebhook) -> None:
|
||||
if item.receiver_type == "alert":
|
||||
await self.repository.complete_webhook(item)
|
||||
return
|
||||
user_id = await self.repository.mapped_user_for_external(item.external_id)
|
||||
if user_id is None:
|
||||
await self.repository.complete_webhook(item)
|
||||
return
|
||||
workflow_id = uuid.uuid4()
|
||||
async with self.repository.transaction() as connection:
|
||||
await connection.execute(
|
||||
text(
|
||||
"""
|
||||
INSERT INTO bitrix_sync.workflow_instances
|
||||
(id,workflow_type,user_id,external_id,state,current_step,deadline_at,
|
||||
created_at,updated_at)
|
||||
VALUES (:id,'contact.webhook',:user_id,:external_id,'running',
|
||||
'read_contact',now()+interval '24 hours',now(),now())
|
||||
"""
|
||||
),
|
||||
{"id": workflow_id, "user_id": user_id, "external_id": item.external_id},
|
||||
)
|
||||
try:
|
||||
result = await self._command(
|
||||
workflow_id,
|
||||
"contact_get",
|
||||
"crm.contact.get",
|
||||
{"id": item.external_id, "select": self._select_fields()},
|
||||
mutating=False,
|
||||
)
|
||||
contact = result.result
|
||||
if str(contact.get(self.settings.contact_user_id_field)) != str(user_id):
|
||||
await self._alert(
|
||||
workflow_id,
|
||||
user_id,
|
||||
"mapping_identity_mismatch",
|
||||
[item.external_id],
|
||||
)
|
||||
await self._manual(workflow_id, "mapping_identity_mismatch", user_id)
|
||||
await self.repository.complete_webhook(item)
|
||||
return
|
||||
citizenship = await self._resolve_citizenship(
|
||||
workflow_id, contact.get(self.settings.contact_citizenship_field)
|
||||
)
|
||||
source_value = contact.get("DATE_MODIFY") or contact.get("updatedTime")
|
||||
await self.repository.apply_crm_profile(
|
||||
user_id,
|
||||
item.external_id,
|
||||
full_name=contact.get("NAME") or None,
|
||||
citizenship=citizenship,
|
||||
email=select_email(contact.get("EMAIL") or []),
|
||||
source_updated_at=parse_crm_datetime(source_value) if source_value else None,
|
||||
source="reconciliation"
|
||||
if item.event_type == "contact.reconciliation"
|
||||
else "webhook",
|
||||
)
|
||||
await self.repository.complete_webhook(item)
|
||||
async with self.repository.transaction() as connection:
|
||||
await connection.execute(
|
||||
text(
|
||||
"""
|
||||
UPDATE bitrix_sync.workflow_instances
|
||||
SET state='succeeded',current_step='done',
|
||||
completed_at=now(),updated_at=now()
|
||||
WHERE id=:id
|
||||
"""
|
||||
),
|
||||
{"id": workflow_id},
|
||||
)
|
||||
except RetryableWorkflow:
|
||||
raise
|
||||
|
||||
async def _map_or_create(self, workflow_id: uuid.UUID, profile: Profile) -> None:
|
||||
if profile.identity_status != "A" or profile.profile_status != "A":
|
||||
await self._deactivate(workflow_id, profile)
|
||||
return
|
||||
validate_phone(profile.phone)
|
||||
if mapped := await self.repository.active_mapping(profile.user_id):
|
||||
await self._ensure_registered(workflow_id, mapped, profile.user_id)
|
||||
return
|
||||
|
||||
found = await self._command(
|
||||
workflow_id,
|
||||
"duplicate_find",
|
||||
"crm.duplicate.findbycomm",
|
||||
{"type": "PHONE", "values": [profile.phone], "entity_type": "CONTACT"},
|
||||
mutating=False,
|
||||
)
|
||||
ids = _contact_ids(found.result)
|
||||
contacts: list[dict[str, Any]] = []
|
||||
for contact_id in ids:
|
||||
result = await self._command(
|
||||
workflow_id,
|
||||
"contact_get",
|
||||
"crm.contact.get",
|
||||
{"id": contact_id, "select": self._select_fields()},
|
||||
mutating=False,
|
||||
)
|
||||
if isinstance(result.result, dict):
|
||||
contacts.append(result.result)
|
||||
|
||||
same_user = next(
|
||||
(
|
||||
item
|
||||
for item in contacts
|
||||
if str(item.get(self.settings.contact_user_id_field)) == str(profile.user_id)
|
||||
),
|
||||
None,
|
||||
)
|
||||
alert_code: str | None = None
|
||||
if same_user:
|
||||
selected_id = str(same_user["ID"])
|
||||
elif not contacts:
|
||||
selected_id = await self._create_contact(workflow_id, profile)
|
||||
else:
|
||||
candidates = [
|
||||
ContactCandidate(
|
||||
str(item["ID"]),
|
||||
parse_crm_datetime(item.get("CREATED_TIME", "1970-01-01T00:00:00Z")),
|
||||
item.get(self.settings.contact_user_id_field),
|
||||
)
|
||||
for item in contacts
|
||||
]
|
||||
selected = choose_newest(candidates)
|
||||
assert selected is not None
|
||||
all_ids = [item.b24_id for item in candidates]
|
||||
if selected.crm_user_id and selected.crm_user_id != str(profile.user_id):
|
||||
selected_id = await self._create_contact(workflow_id, profile)
|
||||
alert_code = "contact_owned_by_other_user"
|
||||
else:
|
||||
selected_id = selected.b24_id
|
||||
await self._write_identity(workflow_id, selected_id, profile.user_id, active=True)
|
||||
if len(candidates) > 1:
|
||||
alert_code = "duplicate_contacts"
|
||||
if alert_code:
|
||||
await self._alert(workflow_id, profile.user_id, alert_code, all_ids)
|
||||
await self._activate_mapping(workflow_id, profile.user_id, selected_id)
|
||||
|
||||
async def _create_contact(self, workflow_id: uuid.UUID, profile: Profile) -> str:
|
||||
result = await self._command(
|
||||
workflow_id,
|
||||
"contact_add",
|
||||
"crm.contact.add",
|
||||
{
|
||||
"fields": {
|
||||
"PHONE": [{"VALUE": profile.phone, "VALUE_TYPE": "WORK"}],
|
||||
self.settings.contact_user_id_field: str(profile.user_id),
|
||||
self.settings.contact_registered_field: "1",
|
||||
}
|
||||
},
|
||||
mutating=True,
|
||||
)
|
||||
return str(result.result)
|
||||
|
||||
async def _update(self, workflow_id: uuid.UUID, profile: Profile) -> None:
|
||||
validate_phone(profile.phone)
|
||||
mapping = await self.repository.active_mapping(profile.user_id)
|
||||
if mapping is None:
|
||||
await self._coalesce_map_or_create(profile.user_id)
|
||||
return
|
||||
await self._command(
|
||||
workflow_id,
|
||||
"contact_update",
|
||||
"crm.contact.update",
|
||||
{
|
||||
"id": mapping,
|
||||
"fields": {"PHONE": [{"VALUE": profile.phone, "VALUE_TYPE": "WORK"}]},
|
||||
},
|
||||
mutating=True,
|
||||
)
|
||||
|
||||
async def _deactivate(self, workflow_id: uuid.UUID, profile: Profile) -> None:
|
||||
mapping = await self.repository.active_mapping(profile.user_id)
|
||||
if mapping is None:
|
||||
return
|
||||
await self._write_identity(workflow_id, mapping, profile.user_id, active=False)
|
||||
async with self.repository.transaction() as connection:
|
||||
await connection.execute(
|
||||
text(
|
||||
"""
|
||||
UPDATE bitrix_sync.entity_external_mapping
|
||||
SET status='closed', closed_at=now(), close_reason='deactivated',
|
||||
workflow_id=:workflow_id, updated_at=now()
|
||||
WHERE entity_id=:user_id AND external_id=:external_id AND status='active'
|
||||
"""
|
||||
),
|
||||
{
|
||||
"workflow_id": workflow_id,
|
||||
"user_id": profile.user_id,
|
||||
"external_id": mapping,
|
||||
},
|
||||
)
|
||||
|
||||
async def process_rebind(self, request_id: uuid.UUID) -> None:
|
||||
async with self.repository.transaction() as connection:
|
||||
request = (
|
||||
await connection.execute(
|
||||
text(
|
||||
"""
|
||||
SELECT id,user_id,old_external_id,target_external_id,workflow_id
|
||||
FROM bitrix_sync.rebind_requests
|
||||
WHERE id=:id AND status IN ('pending','retry_wait')
|
||||
FOR UPDATE
|
||||
"""
|
||||
),
|
||||
{"id": request_id},
|
||||
)
|
||||
).mappings().first()
|
||||
if not request:
|
||||
return
|
||||
await connection.execute(
|
||||
text(
|
||||
"""
|
||||
UPDATE bitrix_sync.rebind_requests
|
||||
SET status='processing',updated_at=now() WHERE id=:id
|
||||
"""
|
||||
),
|
||||
{"id": request_id},
|
||||
)
|
||||
target = await self._command(
|
||||
request["workflow_id"],
|
||||
"rebind_target_get",
|
||||
"crm.contact.get",
|
||||
{"id": request["target_external_id"], "select": self._select_fields()},
|
||||
mutating=False,
|
||||
)
|
||||
target_user = target.result.get(self.settings.contact_user_id_field)
|
||||
if target_user and target_user != str(request["user_id"]):
|
||||
raise BusinessConflict("rebind_target_owned", [request["target_external_id"]])
|
||||
await self._write_identity(
|
||||
request["workflow_id"], request["target_external_id"], request["user_id"], active=True
|
||||
)
|
||||
if request["old_external_id"]:
|
||||
old = await self._command(
|
||||
request["workflow_id"],
|
||||
"rebind_old_get",
|
||||
"crm.contact.get",
|
||||
{"id": request["old_external_id"], "select": self._select_fields()},
|
||||
mutating=False,
|
||||
)
|
||||
if old.result.get(self.settings.contact_user_id_field) == str(request["user_id"]):
|
||||
await self._write_identity(
|
||||
request["workflow_id"],
|
||||
request["old_external_id"],
|
||||
request["user_id"],
|
||||
active=False,
|
||||
)
|
||||
async with self.repository.transaction() as connection:
|
||||
await connection.execute(
|
||||
text(
|
||||
"""
|
||||
UPDATE bitrix_sync.entity_external_mapping
|
||||
SET status='closed',closed_at=now(),close_reason='rebind',
|
||||
workflow_id=:workflow_id,updated_at=now()
|
||||
WHERE entity_id=:user_id AND status='active'
|
||||
"""
|
||||
),
|
||||
{
|
||||
"workflow_id": request["workflow_id"],
|
||||
"user_id": request["user_id"],
|
||||
},
|
||||
)
|
||||
await connection.execute(
|
||||
text(
|
||||
"""
|
||||
INSERT INTO bitrix_sync.entity_external_mapping
|
||||
(id,entity_type,entity_id,external_system,external_entity_type,
|
||||
external_id,status,opened_at,workflow_id,created_at,updated_at)
|
||||
VALUES (gen_random_uuid(),'contact',:user_id,'bitrix24','contact',
|
||||
:target,'active',now(),:workflow_id,now(),now())
|
||||
"""
|
||||
),
|
||||
{
|
||||
"workflow_id": request["workflow_id"],
|
||||
"user_id": request["user_id"],
|
||||
"target": request["target_external_id"],
|
||||
},
|
||||
)
|
||||
await connection.execute(
|
||||
text(
|
||||
"""
|
||||
UPDATE bitrix_sync.rebind_requests
|
||||
SET status='succeeded',completed_at=now(),updated_at=now()
|
||||
WHERE id=:request_id
|
||||
"""
|
||||
),
|
||||
{
|
||||
"request_id": request_id,
|
||||
},
|
||||
)
|
||||
|
||||
async def _ensure_registered(
|
||||
self, workflow_id: uuid.UUID, external_id: str, user_id: uuid.UUID
|
||||
) -> None:
|
||||
contact = await self._command(
|
||||
workflow_id,
|
||||
"contact_get",
|
||||
"crm.contact.get",
|
||||
{"id": external_id, "select": self._select_fields()},
|
||||
mutating=False,
|
||||
)
|
||||
if str(contact.result.get(self.settings.contact_registered_field)) not in {"1", "Y"}:
|
||||
await self._write_identity(workflow_id, external_id, user_id, active=True)
|
||||
|
||||
async def _resolve_citizenship(
|
||||
self, workflow_id: uuid.UUID, enum_id: str | int | None
|
||||
) -> str | None:
|
||||
if enum_id in (None, ""):
|
||||
return None
|
||||
async with self.repository.transaction() as connection:
|
||||
value = (
|
||||
await connection.execute(
|
||||
text(
|
||||
"""
|
||||
SELECT display_value FROM bitrix_sync.citizenship_dictionary
|
||||
WHERE enum_id=:enum_id AND expires_at>now()
|
||||
"""
|
||||
),
|
||||
{"enum_id": str(enum_id)},
|
||||
)
|
||||
).scalar_one_or_none()
|
||||
if value is not None:
|
||||
return value
|
||||
result = await self._command(
|
||||
workflow_id,
|
||||
"citizenship_fields_get",
|
||||
"crm.contact.userfield.list",
|
||||
{"filter": {"FIELD_NAME": self.settings.contact_citizenship_field}},
|
||||
mutating=False,
|
||||
)
|
||||
fields = result.result if isinstance(result.result, list) else []
|
||||
entries = fields[0].get("LIST", []) if fields else []
|
||||
async with self.repository.transaction() as connection:
|
||||
for entry in entries:
|
||||
if entry.get("ID") is None or not isinstance(entry.get("VALUE"), str):
|
||||
continue
|
||||
await connection.execute(
|
||||
text(
|
||||
"""
|
||||
INSERT INTO bitrix_sync.citizenship_dictionary
|
||||
(enum_id,display_value,loaded_at,expires_at)
|
||||
VALUES (:id,:value,now(),now()+interval '1 hour')
|
||||
ON CONFLICT (enum_id) DO UPDATE
|
||||
SET display_value=excluded.display_value,loaded_at=now(),
|
||||
expires_at=excluded.expires_at
|
||||
"""
|
||||
),
|
||||
{"id": str(entry["ID"]), "value": entry["VALUE"]},
|
||||
)
|
||||
match = next(
|
||||
(
|
||||
entry["VALUE"]
|
||||
for entry in entries
|
||||
if str(entry.get("ID")) == str(enum_id)
|
||||
),
|
||||
None,
|
||||
)
|
||||
if match is None:
|
||||
raise BusinessConflict("unknown_citizenship_enum", [str(enum_id)])
|
||||
return match
|
||||
|
||||
async def _write_identity(
|
||||
self, workflow_id: uuid.UUID, external_id: str, user_id: uuid.UUID, *, active: bool
|
||||
) -> None:
|
||||
fields: dict[str, Any] = {self.settings.contact_registered_field: "1" if active else "0"}
|
||||
fields[self.settings.contact_user_id_field] = str(user_id) if active else ""
|
||||
await self._command(
|
||||
workflow_id,
|
||||
"contact_update",
|
||||
"crm.contact.update",
|
||||
{"id": external_id, "fields": fields},
|
||||
mutating=True,
|
||||
)
|
||||
|
||||
async def _command(
|
||||
self,
|
||||
workflow_id: uuid.UUID,
|
||||
command_type: str,
|
||||
method: str,
|
||||
params: dict[str, Any],
|
||||
*,
|
||||
mutating: bool,
|
||||
) -> CrmResult:
|
||||
command_id = uuid.uuid4()
|
||||
async with self.repository.transaction() as connection:
|
||||
await connection.execute(
|
||||
text(
|
||||
"""
|
||||
INSERT INTO bitrix_sync.crm_commands
|
||||
(id,workflow_id,command_type,safe_request,status,attempt_count,
|
||||
next_attempt_at,created_at,updated_at)
|
||||
VALUES (:id,:workflow_id,:type,:safe_request,'in_flight',1,now(),now(),now())
|
||||
"""
|
||||
),
|
||||
{
|
||||
"id": command_id,
|
||||
"workflow_id": workflow_id,
|
||||
"type": command_type,
|
||||
"safe_request": {"keys": sorted(params), "method_class": method.split(".")[-1]},
|
||||
},
|
||||
)
|
||||
while True:
|
||||
limiter_delay = await self.repository.reserve_limiter_token(
|
||||
self.settings.limiter_refill_per_sec, self.settings.limiter_burst
|
||||
)
|
||||
if limiter_delay <= 0:
|
||||
break
|
||||
await asyncio.sleep(limiter_delay)
|
||||
async with self._in_flight:
|
||||
result = await self.crm.call(method, params, mutating=mutating)
|
||||
status = result.outcome.value
|
||||
async with self.repository.transaction() as connection:
|
||||
await connection.execute(
|
||||
text(
|
||||
"""
|
||||
UPDATE bitrix_sync.crm_commands
|
||||
SET status=:status,safe_error_code=:error,http_status=:http_status,
|
||||
safe_response=:safe_response,
|
||||
completed_at=CASE WHEN :terminal THEN now() END,
|
||||
updated_at=now()
|
||||
WHERE id=:id
|
||||
"""
|
||||
),
|
||||
{
|
||||
"id": command_id,
|
||||
"status": status,
|
||||
"error": result.error_code,
|
||||
"http_status": result.http_status,
|
||||
"safe_response": {"has_result": result.result is not None},
|
||||
"terminal": result.outcome in {CrmOutcome.SUCCEEDED, CrmOutcome.PERMANENT},
|
||||
},
|
||||
)
|
||||
if result.outcome == CrmOutcome.SUCCEEDED:
|
||||
return result
|
||||
if result.outcome == CrmOutcome.PERMANENT:
|
||||
await self._technical_failure(workflow_id, result.error_code or "crm_permanent")
|
||||
raise BusinessConflict("technical_configuration_failure")
|
||||
raise RetryableWorkflow(result.error_code or result.outcome.value, result.retry_after)
|
||||
|
||||
async def _activate_mapping(
|
||||
self, workflow_id: uuid.UUID, user_id: uuid.UUID, external_id: str
|
||||
) -> None:
|
||||
async with self.repository.transaction() as connection:
|
||||
await connection.execute(
|
||||
text(
|
||||
"""
|
||||
INSERT INTO bitrix_sync.entity_external_mapping
|
||||
(id,entity_type,entity_id,external_system,external_entity_type,
|
||||
external_id,status,opened_at,workflow_id,created_at,updated_at)
|
||||
VALUES (gen_random_uuid(),'contact',:user_id,'bitrix24','contact',
|
||||
:external_id,'active',now(),:workflow_id,now(),now())
|
||||
ON CONFLICT DO NOTHING
|
||||
"""
|
||||
),
|
||||
{
|
||||
"workflow_id": workflow_id,
|
||||
"user_id": user_id,
|
||||
"external_id": external_id,
|
||||
},
|
||||
)
|
||||
|
||||
async def _coalesce_map_or_create(self, user_id: uuid.UUID) -> None:
|
||||
async with self.repository.transaction() as connection:
|
||||
await connection.execute(
|
||||
text(
|
||||
"""
|
||||
INSERT INTO han_app.sync_queue
|
||||
(id,task_type,entity_type,entity_id,dedup_key,payload_json,status,
|
||||
attempt_count,next_attempt_at,created_at,updated_at)
|
||||
VALUES (gen_random_uuid(),'contact.map_or_create','contact',:user_id,
|
||||
'contact.map_or_create:'||:user_id::text,
|
||||
jsonb_build_object('schema_version',1,'user_id',:user_id),
|
||||
'pending',0,now(),now(),now())
|
||||
ON CONFLICT DO NOTHING
|
||||
"""
|
||||
),
|
||||
{"user_id": user_id},
|
||||
)
|
||||
|
||||
async def _alert(
|
||||
self, workflow_id: uuid.UUID, user_id: uuid.UUID, alert_type: str, candidates: list[str]
|
||||
) -> None:
|
||||
fingerprint = f"{alert_type}:{user_id}"
|
||||
async with self.repository.transaction() as connection:
|
||||
await connection.execute(
|
||||
text(
|
||||
"""
|
||||
INSERT INTO bitrix_sync.business_alerts
|
||||
(id,fingerprint,alert_type,severity,app_user_id,candidate_external_ids,
|
||||
workflow_id,status,occurrence_count,first_occurred_at,last_occurred_at,
|
||||
created_at,updated_at)
|
||||
VALUES (gen_random_uuid(),encode(digest(:fingerprint,'sha256'),'hex'),
|
||||
:type,'warning',:user_id,:candidates,:workflow_id,'open',1,
|
||||
now(),now(),now(),now())
|
||||
ON CONFLICT (alert_type,fingerprint) WHERE status='open'
|
||||
DO UPDATE SET occurrence_count=business_alerts.occurrence_count+1,
|
||||
last_occurred_at=now(),updated_at=now()
|
||||
"""
|
||||
),
|
||||
{
|
||||
"fingerprint": fingerprint,
|
||||
"type": alert_type,
|
||||
"user_id": user_id,
|
||||
"candidates": candidates,
|
||||
"workflow_id": workflow_id,
|
||||
},
|
||||
)
|
||||
|
||||
async def _manual(self, workflow_id: uuid.UUID, code: str, user_id: uuid.UUID) -> None:
|
||||
async with self.repository.transaction() as connection:
|
||||
await connection.execute(
|
||||
text(
|
||||
"""
|
||||
UPDATE bitrix_sync.workflow_instances
|
||||
SET state='waiting_manual',outcome=:code,updated_at=now()
|
||||
WHERE id=:id
|
||||
"""
|
||||
),
|
||||
{"id": workflow_id, "code": code},
|
||||
)
|
||||
|
||||
async def _technical_failure(self, workflow_id: uuid.UUID, code: str) -> None:
|
||||
async with self.repository.transaction() as connection:
|
||||
await connection.execute(
|
||||
text(
|
||||
"""
|
||||
INSERT INTO bitrix_sync.technical_dead_letters
|
||||
(id,workflow_id,operation,safe_error_code,failed_at,created_at)
|
||||
VALUES (gen_random_uuid(),:workflow_id,'crm_command',:code,now(),now())
|
||||
"""
|
||||
),
|
||||
{"workflow_id": workflow_id, "code": code[:64]},
|
||||
)
|
||||
|
||||
def _select_fields(self) -> list[str]:
|
||||
return [
|
||||
"ID",
|
||||
"NAME",
|
||||
"PHONE",
|
||||
"EMAIL",
|
||||
"CREATED_TIME",
|
||||
"DATE_MODIFY",
|
||||
self.settings.contact_user_id_field,
|
||||
self.settings.contact_registered_field,
|
||||
self.settings.contact_citizenship_field,
|
||||
]
|
||||
|
||||
|
||||
class RetryableWorkflow(Exception):
|
||||
def __init__(self, code: str, retry_after: float | None = None) -> None:
|
||||
self.code = code
|
||||
self.retry_after = retry_after
|
||||
super().__init__(code)
|
||||
|
||||
|
||||
def _contact_ids(result: Any) -> list[str]:
|
||||
if isinstance(result, dict):
|
||||
values = result.get("CONTACT", [])
|
||||
else:
|
||||
values = result or []
|
||||
return sorted({str(value) for value in values if str(value).isdigit()}, key=int)
|
||||
@@ -0,0 +1,173 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import hmac
|
||||
from contextlib import asynccontextmanager
|
||||
from typing import Annotated
|
||||
|
||||
import uvicorn
|
||||
from fastapi import Depends, FastAPI, Header, HTTPException, Query, Request, Response
|
||||
from fastapi.responses import JSONResponse
|
||||
from sqlalchemy import text
|
||||
|
||||
from app.config import Settings, load_settings
|
||||
from app.repository import Repository
|
||||
from app.security import WebhookValidationError, parse_bounded_form, validate_webhook
|
||||
|
||||
|
||||
@asynccontextmanager
|
||||
async def lifespan(app: FastAPI):
|
||||
settings = load_settings()
|
||||
app.state.settings = settings
|
||||
app.state.repository = (
|
||||
Repository(settings.database_url.get_secret_value(), settings.db_pool_size)
|
||||
if settings.enabled and settings.database_url
|
||||
else None
|
||||
)
|
||||
yield
|
||||
if app.state.repository:
|
||||
await app.state.repository.close()
|
||||
|
||||
|
||||
app = FastAPI(
|
||||
title="HAN Bitrix Sync",
|
||||
version="0.1.0",
|
||||
docs_url=None,
|
||||
redoc_url=None,
|
||||
lifespan=lifespan,
|
||||
)
|
||||
|
||||
|
||||
def settings(request: Request) -> Settings:
|
||||
return request.app.state.settings
|
||||
|
||||
|
||||
def repository(request: Request) -> Repository:
|
||||
repo = request.app.state.repository
|
||||
if repo is None:
|
||||
raise HTTPException(status_code=503, detail="sync_disabled")
|
||||
return repo
|
||||
|
||||
|
||||
async def require_service_token(
|
||||
request: Request,
|
||||
authorization: Annotated[str | None, Header()] = None,
|
||||
) -> None:
|
||||
configured = settings(request).service_token
|
||||
expected = f"Bearer {configured.get_secret_value()}" if configured else ""
|
||||
if not authorization or not hmac.compare_digest(authorization, expected):
|
||||
raise HTTPException(status_code=401, detail="unauthorized")
|
||||
|
||||
|
||||
@app.get("/health/live", include_in_schema=True)
|
||||
async def live() -> dict[str, str]:
|
||||
return {"status": "live"}
|
||||
|
||||
|
||||
@app.get("/health/ready", include_in_schema=True)
|
||||
async def ready(request: Request) -> Response:
|
||||
config = settings(request)
|
||||
if not config.enabled:
|
||||
return JSONResponse(
|
||||
status_code=503,
|
||||
content={"status": "not_ready", "reason": "sync_disabled"},
|
||||
)
|
||||
repo = repository(request)
|
||||
if not await repo.ping():
|
||||
return JSONResponse(status_code=503, content={"status": "not_ready", "reason": "database"})
|
||||
return JSONResponse({"status": "ready", "mode": config.mode})
|
||||
|
||||
|
||||
@app.get(
|
||||
"/internal/sync/v1/status",
|
||||
dependencies=[Depends(require_service_token)],
|
||||
include_in_schema=True,
|
||||
)
|
||||
async def sync_status(request: Request) -> dict:
|
||||
result = await repository(request).status()
|
||||
result["mode"] = settings(request).mode
|
||||
return result
|
||||
|
||||
|
||||
@app.post("/bitrix/sync/webhook/contact", status_code=202, include_in_schema=True)
|
||||
async def contact_webhook(
|
||||
request: Request,
|
||||
token: Annotated[str | None, Query(max_length=256)] = None,
|
||||
ID: Annotated[str | None, Query(pattern=r"^[1-9][0-9]{0,19}$")] = None, # noqa: N803
|
||||
) -> Response:
|
||||
return await _receive(request, "contact", {"token": token or "", "ID": ID or ""})
|
||||
|
||||
|
||||
@app.post("/bitrix/sync/webhook/alert", status_code=202, include_in_schema=True)
|
||||
async def alert_webhook(
|
||||
request: Request,
|
||||
token: Annotated[str | None, Query(max_length=256)] = None,
|
||||
ID: Annotated[str | None, Query(pattern=r"^[1-9][0-9]{0,19}$")] = None, # noqa: N803
|
||||
) -> Response:
|
||||
return await _receive(request, "alert", {"token": token or "", "ID": ID or ""})
|
||||
|
||||
|
||||
async def _receive(request: Request, receiver: str, query: dict[str, str]) -> Response:
|
||||
config = settings(request)
|
||||
if not config.enabled:
|
||||
raise HTTPException(status_code=503, detail="sync_disabled")
|
||||
if request.headers.get("content-type", "").split(";", 1)[0].lower() != (
|
||||
"application/x-www-form-urlencoded"
|
||||
):
|
||||
raise HTTPException(status_code=400, detail="invalid_content_type")
|
||||
content_length = request.headers.get("content-length")
|
||||
if content_length and (
|
||||
not content_length.isdigit() or int(content_length) > config.webhook_max_body_bytes
|
||||
):
|
||||
raise HTTPException(status_code=413, detail="body_too_large")
|
||||
body = await request.body()
|
||||
if len(body) > config.webhook_max_body_bytes:
|
||||
raise HTTPException(status_code=413, detail="body_too_large")
|
||||
form = parse_bounded_form(body, max_fields=config.webhook_max_fields)
|
||||
# The container is reachable only from the trusted VM2 nginx network.
|
||||
# nginx overwrites X-Real-IP from the TCP peer after its CIDR check.
|
||||
source_ip = request.headers.get("x-real-ip") or (request.client.host if request.client else "")
|
||||
alert_entity_type_id = (
|
||||
await _alert_entity_type(repository(request)) if receiver == "alert" else None
|
||||
)
|
||||
try:
|
||||
event = validate_webhook(
|
||||
receiver,
|
||||
query,
|
||||
form,
|
||||
source_ip,
|
||||
config,
|
||||
alert_entity_type_id=alert_entity_type_id,
|
||||
)
|
||||
except PermissionError as exc:
|
||||
raise HTTPException(status_code=403, detail="forbidden") from exc
|
||||
except WebhookValidationError as exc:
|
||||
raise HTTPException(status_code=400, detail="malformed_webhook") from exc
|
||||
await repository(request).insert_webhook(
|
||||
event.receiver_type, event.event_type, event.entity_id, event.source_ip
|
||||
)
|
||||
return Response(status_code=202)
|
||||
|
||||
|
||||
async def _alert_entity_type(repo: Repository) -> int | None:
|
||||
async with repo.engine.connect() as connection:
|
||||
value = (
|
||||
await connection.execute(
|
||||
text(
|
||||
"""
|
||||
SELECT (value_json->>'entity_type_id')::integer
|
||||
FROM bitrix_sync.settings
|
||||
WHERE key='business_alerts' AND active=true AND validation_status='valid'
|
||||
"""
|
||||
)
|
||||
)
|
||||
).scalar_one_or_none()
|
||||
return value
|
||||
|
||||
|
||||
def run() -> None:
|
||||
uvicorn.run(
|
||||
"app.main:app",
|
||||
host="0.0.0.0", # noqa: S104 - container-only port, not host-published
|
||||
port=8080,
|
||||
proxy_headers=False,
|
||||
)
|
||||
@@ -0,0 +1,56 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from typing import Any
|
||||
|
||||
from app.domain import select_email
|
||||
|
||||
|
||||
class UnknownCitizenship(ValueError):
|
||||
pass
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class CitizenshipEntry:
|
||||
enum_id: str
|
||||
display_value: str
|
||||
loaded_at: datetime
|
||||
|
||||
|
||||
class CitizenshipDictionary:
|
||||
def __init__(self, ttl_seconds: int = 3600) -> None:
|
||||
self.ttl = timedelta(seconds=ttl_seconds)
|
||||
self._entries: dict[str, CitizenshipEntry] = {}
|
||||
self.loaded_at: datetime | None = None
|
||||
|
||||
def load(self, values: list[dict[str, Any]], now: datetime | None = None) -> None:
|
||||
loaded_at = now or datetime.now(UTC)
|
||||
self._entries = {
|
||||
str(item["ID"]): CitizenshipEntry(
|
||||
str(item["ID"]), str(item["VALUE"]), loaded_at
|
||||
)
|
||||
for item in values
|
||||
if item.get("ID") is not None and isinstance(item.get("VALUE"), str)
|
||||
}
|
||||
self.loaded_at = loaded_at
|
||||
|
||||
def resolve(self, enum_id: str | int | None, now: datetime | None = None) -> str | None:
|
||||
if enum_id in (None, ""):
|
||||
return None
|
||||
entry = self._entries.get(str(enum_id))
|
||||
if entry is None:
|
||||
raise UnknownCitizenship(str(enum_id))
|
||||
if (now or datetime.now(UTC)) - entry.loaded_at > self.ttl:
|
||||
raise UnknownCitizenship(str(enum_id))
|
||||
return entry.display_value
|
||||
|
||||
|
||||
def crm_master_projection(
|
||||
contact: dict[str, Any], citizenship: CitizenshipDictionary, citizenship_field: str
|
||||
) -> dict[str, str | None]:
|
||||
return {
|
||||
"full_name": contact.get("NAME") or None,
|
||||
"email": select_email(contact.get("EMAIL") or []),
|
||||
"citizenship": citizenship.resolve(contact.get(citizenship_field)),
|
||||
}
|
||||
@@ -0,0 +1,139 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import text
|
||||
|
||||
from app.config import load_settings
|
||||
from app.crm import CrmClient, CrmOutcome
|
||||
from app.repository import Repository
|
||||
|
||||
|
||||
class IncrementalReconciler:
|
||||
def __init__(self, repository: Repository, crm: CrmClient, registered_rest_field: str) -> None:
|
||||
self.repository = repository
|
||||
self.crm = crm
|
||||
self.registered_rest_field = registered_rest_field
|
||||
|
||||
async def run_once(self, overlap_seconds: int) -> int:
|
||||
async with self.repository.transaction() as connection:
|
||||
acquired = (
|
||||
await connection.execute(
|
||||
text(
|
||||
"""
|
||||
SELECT pg_try_advisory_xact_lock(
|
||||
hashtext('bitrix-contact-reconciliation')
|
||||
)
|
||||
"""
|
||||
)
|
||||
)
|
||||
).scalar_one()
|
||||
if not acquired:
|
||||
return 0
|
||||
cursor = (
|
||||
await connection.execute(
|
||||
text(
|
||||
"""
|
||||
SELECT watermark FROM bitrix_sync.reconciliation_cursors
|
||||
WHERE job_type='contact_incremental' FOR UPDATE
|
||||
"""
|
||||
)
|
||||
)
|
||||
).scalar_one_or_none()
|
||||
started_at = datetime.now(UTC)
|
||||
since = (cursor or datetime(1970, 1, 1, tzinfo=UTC)) - timedelta(seconds=overlap_seconds)
|
||||
start = 0
|
||||
scanned: list[str] = []
|
||||
while True:
|
||||
result = await self.crm.call(
|
||||
"crm.item.list",
|
||||
{
|
||||
"entityTypeId": 3,
|
||||
"select": ["id"],
|
||||
"filter": {
|
||||
">=updatedTime": since.replace(tzinfo=None).isoformat(timespec="seconds"),
|
||||
"opened": 1,
|
||||
self.registered_rest_field: 1,
|
||||
},
|
||||
"start": start,
|
||||
},
|
||||
mutating=False,
|
||||
)
|
||||
if result.outcome != CrmOutcome.SUCCEEDED:
|
||||
raise RuntimeError(result.error_code or "reconciliation_failed")
|
||||
payload: dict[str, Any] = result.result or {}
|
||||
items = payload.get("items", payload if isinstance(payload, list) else [])
|
||||
scanned.extend(str(item["id"]) for item in items if "id" in item)
|
||||
next_start = payload.get("next")
|
||||
if next_start is None:
|
||||
break
|
||||
start = int(next_start)
|
||||
await self._enqueue_changed(scanned)
|
||||
async with self.repository.transaction() as connection:
|
||||
await connection.execute(
|
||||
text(
|
||||
"""
|
||||
INSERT INTO bitrix_sync.reconciliation_cursors
|
||||
(job_type,watermark,overlap_seconds,last_success_at,last_scanned_count,
|
||||
created_at,updated_at)
|
||||
VALUES ('contact_incremental',:watermark,:overlap,now(),:count,now(),now())
|
||||
ON CONFLICT (job_type) DO UPDATE
|
||||
SET watermark=excluded.watermark,overlap_seconds=excluded.overlap_seconds,
|
||||
last_success_at=now(),last_scanned_count=excluded.last_scanned_count,
|
||||
updated_at=now()
|
||||
"""
|
||||
),
|
||||
{"watermark": started_at, "overlap": overlap_seconds, "count": len(scanned)},
|
||||
)
|
||||
return len(scanned)
|
||||
|
||||
async def _enqueue_changed(self, external_ids: list[str]) -> None:
|
||||
if not external_ids:
|
||||
return
|
||||
async with self.repository.transaction() as connection:
|
||||
await connection.execute(
|
||||
text(
|
||||
"""
|
||||
INSERT INTO bitrix_sync.webhook_inbox
|
||||
(id,receiver_type,event_type,external_entity_id,status,coalesced_count,
|
||||
received_at,last_received_at)
|
||||
SELECT gen_random_uuid(),'contact','contact.reconciliation',id,'received',1,
|
||||
now(),now()
|
||||
FROM unnest(CAST(:ids AS text[])) id
|
||||
WHERE NOT EXISTS (
|
||||
SELECT 1 FROM bitrix_sync.webhook_inbox w
|
||||
WHERE w.receiver_type='contact' AND w.external_entity_id=id
|
||||
AND w.status IN ('received','processing','retry_wait')
|
||||
)
|
||||
"""
|
||||
),
|
||||
{"ids": external_ids},
|
||||
)
|
||||
|
||||
|
||||
async def reconciliation_main() -> None:
|
||||
settings = load_settings()
|
||||
if not settings.enabled:
|
||||
return
|
||||
assert settings.database_url and settings.crm_rest_webhook_url and settings.portal_host
|
||||
assert settings.contact_registered_field
|
||||
repository = Repository(settings.database_url.get_secret_value(), settings.db_pool_size)
|
||||
crm = CrmClient(
|
||||
settings.crm_rest_webhook_url.get_secret_value(),
|
||||
settings.portal_host,
|
||||
settings.http_timeout_sec,
|
||||
)
|
||||
reconciler = IncrementalReconciler(
|
||||
repository, crm, settings.rest_field_name(settings.contact_registered_field)
|
||||
)
|
||||
try:
|
||||
await reconciler.run_once(settings.reconciliation_overlap_seconds)
|
||||
finally:
|
||||
await crm.close()
|
||||
await repository.close()
|
||||
|
||||
|
||||
def run() -> None:
|
||||
asyncio.run(reconciliation_main())
|
||||
@@ -0,0 +1,533 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import ssl
|
||||
import uuid
|
||||
from collections.abc import AsyncIterator
|
||||
from contextlib import asynccontextmanager
|
||||
from dataclasses import dataclass
|
||||
from datetime import UTC, datetime
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import text
|
||||
from sqlalchemy.ext.asyncio import AsyncConnection, AsyncEngine, create_async_engine
|
||||
|
||||
|
||||
def postgres_ssl_context() -> ssl.SSLContext:
|
||||
ca_file = os.environ.get("PG_CA_FILE")
|
||||
if not ca_file:
|
||||
raise RuntimeError("PG_CA_FILE is required")
|
||||
context = ssl.create_default_context(cafile=ca_file)
|
||||
context.check_hostname = True
|
||||
context.verify_mode = ssl.CERT_REQUIRED
|
||||
return context
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class LeasedTask:
|
||||
id: uuid.UUID
|
||||
task_type: str
|
||||
user_id: uuid.UUID
|
||||
lease_token: uuid.UUID
|
||||
attempt_count: int
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class LeasedWebhook:
|
||||
id: uuid.UUID
|
||||
receiver_type: str
|
||||
event_type: str
|
||||
external_id: str
|
||||
lease_token: uuid.UUID
|
||||
attempt_count: int
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Profile:
|
||||
user_id: uuid.UUID
|
||||
phone: str
|
||||
identity_status: str
|
||||
profile_status: str
|
||||
|
||||
|
||||
class Repository:
|
||||
def __init__(self, database_url: str, pool_size: int = 5) -> None:
|
||||
self.engine: AsyncEngine = create_async_engine(
|
||||
database_url,
|
||||
pool_size=pool_size,
|
||||
pool_pre_ping=True,
|
||||
connect_args={"ssl": postgres_ssl_context()},
|
||||
)
|
||||
|
||||
async def close(self) -> None:
|
||||
await self.engine.dispose()
|
||||
|
||||
@asynccontextmanager
|
||||
async def transaction(self) -> AsyncIterator[AsyncConnection]:
|
||||
async with self.engine.begin() as connection:
|
||||
yield connection
|
||||
|
||||
async def ping(self) -> bool:
|
||||
try:
|
||||
async with self.engine.connect() as connection:
|
||||
await connection.execute(text("SELECT 1"))
|
||||
return True
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
async def claim_tasks(
|
||||
self, worker_id: str, limit: int, lease_seconds: int
|
||||
) -> list[LeasedTask]:
|
||||
sql = text(
|
||||
"""
|
||||
WITH candidates AS (
|
||||
SELECT id
|
||||
FROM han_app.sync_queue
|
||||
WHERE status IN ('pending','retry_wait')
|
||||
AND next_attempt_at <= now()
|
||||
AND (locked_until IS NULL OR locked_until < now())
|
||||
ORDER BY next_attempt_at, created_at
|
||||
FOR UPDATE SKIP LOCKED
|
||||
LIMIT :limit
|
||||
)
|
||||
UPDATE han_app.sync_queue q
|
||||
SET status='leased', locked_by=:worker_id,
|
||||
locked_until=now() + make_interval(secs => :lease_seconds),
|
||||
lease_token=gen_random_uuid(), updated_at=now()
|
||||
FROM candidates c
|
||||
WHERE q.id=c.id
|
||||
RETURNING q.id, q.task_type, q.entity_id, q.lease_token, q.attempt_count
|
||||
"""
|
||||
)
|
||||
async with self.engine.begin() as connection:
|
||||
rows = (
|
||||
await connection.execute(
|
||||
sql,
|
||||
{"worker_id": worker_id, "limit": limit, "lease_seconds": lease_seconds},
|
||||
)
|
||||
).mappings()
|
||||
return [
|
||||
LeasedTask(
|
||||
row["id"],
|
||||
row["task_type"],
|
||||
row["entity_id"],
|
||||
row["lease_token"],
|
||||
row["attempt_count"],
|
||||
)
|
||||
for row in rows
|
||||
]
|
||||
|
||||
async def load_profile(self, user_id: uuid.UUID) -> Profile | None:
|
||||
sql = text(
|
||||
"""
|
||||
SELECT i.id user_id, i.phone_number phone, i.record_status identity_status,
|
||||
p.record_status profile_status
|
||||
FROM han_app.user_identities i
|
||||
JOIN han_app.client_profiles p ON p.user_id=i.id
|
||||
WHERE i.id=:user_id
|
||||
"""
|
||||
)
|
||||
async with self.engine.connect() as connection:
|
||||
row = (await connection.execute(sql, {"user_id": user_id})).mappings().first()
|
||||
return Profile(**row) if row else None
|
||||
|
||||
async def active_mapping(self, user_id: uuid.UUID) -> str | None:
|
||||
sql = text(
|
||||
"""
|
||||
SELECT external_id FROM bitrix_sync.entity_external_mapping
|
||||
WHERE external_system='bitrix24' AND entity_type='contact'
|
||||
AND entity_id=:user_id AND status='active'
|
||||
"""
|
||||
)
|
||||
async with self.engine.connect() as connection:
|
||||
return (await connection.execute(sql, {"user_id": user_id})).scalar_one_or_none()
|
||||
|
||||
async def create_workflow(self, task: LeasedTask) -> uuid.UUID:
|
||||
workflow_id = uuid.uuid4()
|
||||
sql = text(
|
||||
"""
|
||||
INSERT INTO bitrix_sync.workflow_instances
|
||||
(id, workflow_type, user_id, state, current_step, source_task_id,
|
||||
deadline_at, created_at, updated_at)
|
||||
VALUES (:id, :workflow_type, :user_id, 'created', 'load_profile', :task_id,
|
||||
now() + interval '24 hours', now(), now())
|
||||
ON CONFLICT (source_task_id) DO UPDATE SET updated_at=now()
|
||||
RETURNING id
|
||||
"""
|
||||
)
|
||||
async with self.engine.begin() as connection:
|
||||
return (
|
||||
await connection.execute(
|
||||
sql,
|
||||
{
|
||||
"id": workflow_id,
|
||||
"workflow_type": task.task_type,
|
||||
"user_id": task.user_id,
|
||||
"task_id": task.id,
|
||||
},
|
||||
)
|
||||
).scalar_one()
|
||||
|
||||
async def complete_task(self, task: LeasedTask, workflow_id: uuid.UUID) -> bool:
|
||||
async with self.engine.begin() as connection:
|
||||
result = await connection.execute(
|
||||
text(
|
||||
"""
|
||||
UPDATE han_app.sync_queue
|
||||
SET status='processed', completed_at=now(), locked_by=NULL,
|
||||
locked_until=NULL, lease_token=NULL, updated_at=now()
|
||||
WHERE id=:id AND status='leased' AND lease_token=:lease_token
|
||||
"""
|
||||
),
|
||||
{"id": task.id, "lease_token": task.lease_token},
|
||||
)
|
||||
if result.rowcount != 1:
|
||||
return False
|
||||
await connection.execute(
|
||||
text(
|
||||
"""
|
||||
UPDATE bitrix_sync.workflow_instances
|
||||
SET state='succeeded', current_step='done', outcome='processed',
|
||||
completed_at=now(), updated_at=now()
|
||||
WHERE id=:workflow_id AND state NOT IN ('succeeded','failed','cancelled')
|
||||
"""
|
||||
),
|
||||
{"workflow_id": workflow_id},
|
||||
)
|
||||
return True
|
||||
|
||||
async def retry_task(
|
||||
self, task: LeasedTask, safe_code: str, delay_seconds: float
|
||||
) -> bool:
|
||||
sql = text(
|
||||
"""
|
||||
UPDATE han_app.sync_queue
|
||||
SET status='retry_wait', attempt_count=attempt_count+1,
|
||||
next_attempt_at=now() + make_interval(secs => :delay),
|
||||
last_error_code=:code, last_error_at=now(),
|
||||
locked_by=NULL, locked_until=NULL, lease_token=NULL, updated_at=now()
|
||||
WHERE id=:id AND status='leased' AND lease_token=:lease_token
|
||||
"""
|
||||
)
|
||||
async with self.engine.begin() as connection:
|
||||
result = await connection.execute(
|
||||
sql,
|
||||
{
|
||||
"id": task.id,
|
||||
"lease_token": task.lease_token,
|
||||
"code": safe_code[:64],
|
||||
"delay": delay_seconds,
|
||||
},
|
||||
)
|
||||
return result.rowcount == 1
|
||||
|
||||
async def insert_webhook(
|
||||
self,
|
||||
receiver_type: str,
|
||||
event_type: str,
|
||||
entity_id: str,
|
||||
source_ip: str,
|
||||
) -> uuid.UUID:
|
||||
inbox_id = uuid.uuid4()
|
||||
async with self.engine.begin() as connection:
|
||||
existing = await connection.execute(
|
||||
text(
|
||||
"""
|
||||
SELECT id FROM bitrix_sync.webhook_inbox
|
||||
WHERE receiver_type=:receiver AND external_entity_id=:entity_id
|
||||
AND status IN ('received','processing','retry_wait')
|
||||
FOR UPDATE
|
||||
"""
|
||||
),
|
||||
{"receiver": receiver_type, "entity_id": entity_id},
|
||||
)
|
||||
if row := existing.first():
|
||||
await connection.execute(
|
||||
text(
|
||||
"""
|
||||
UPDATE bitrix_sync.webhook_inbox
|
||||
SET coalesced_count=coalesced_count+1, last_received_at=now()
|
||||
WHERE id=:id
|
||||
"""
|
||||
),
|
||||
{"id": row[0]},
|
||||
)
|
||||
return row[0]
|
||||
await connection.execute(
|
||||
text(
|
||||
"""
|
||||
INSERT INTO bitrix_sync.webhook_inbox
|
||||
(id, receiver_type, event_type, external_entity_id, source_ip,
|
||||
status, coalesced_count, received_at, last_received_at)
|
||||
VALUES (:id,:receiver,:event,:entity_id,CAST(:source_ip AS inet),
|
||||
'received',1,now(),now())
|
||||
"""
|
||||
),
|
||||
{
|
||||
"id": inbox_id,
|
||||
"receiver": receiver_type,
|
||||
"event": event_type,
|
||||
"entity_id": entity_id,
|
||||
"source_ip": source_ip,
|
||||
},
|
||||
)
|
||||
return inbox_id
|
||||
|
||||
async def claim_webhooks(
|
||||
self, worker_id: str, limit: int, lease_seconds: int
|
||||
) -> list[LeasedWebhook]:
|
||||
sql = text(
|
||||
"""
|
||||
WITH candidates AS (
|
||||
SELECT id FROM bitrix_sync.webhook_inbox
|
||||
WHERE status IN ('received','retry_wait') AND next_attempt_at<=now()
|
||||
AND (locked_until IS NULL OR locked_until<now())
|
||||
ORDER BY next_attempt_at,received_at
|
||||
FOR UPDATE SKIP LOCKED LIMIT :limit
|
||||
)
|
||||
UPDATE bitrix_sync.webhook_inbox w
|
||||
SET status='processing',locked_by=:worker_id,
|
||||
locked_until=now()+make_interval(secs=>:lease_seconds),
|
||||
lease_token=gen_random_uuid()
|
||||
FROM candidates c WHERE w.id=c.id
|
||||
RETURNING w.id,w.receiver_type,w.event_type,w.external_entity_id,
|
||||
w.lease_token,w.attempt_count
|
||||
"""
|
||||
)
|
||||
async with self.engine.begin() as connection:
|
||||
rows = (
|
||||
await connection.execute(
|
||||
sql,
|
||||
{"worker_id": worker_id, "limit": limit, "lease_seconds": lease_seconds},
|
||||
)
|
||||
).mappings()
|
||||
return [
|
||||
LeasedWebhook(
|
||||
row["id"],
|
||||
row["receiver_type"],
|
||||
row["event_type"],
|
||||
row["external_entity_id"],
|
||||
row["lease_token"],
|
||||
row["attempt_count"],
|
||||
)
|
||||
for row in rows
|
||||
]
|
||||
|
||||
async def mapped_user_for_external(self, external_id: str) -> uuid.UUID | None:
|
||||
async with self.engine.connect() as connection:
|
||||
return (
|
||||
await connection.execute(
|
||||
text(
|
||||
"""
|
||||
SELECT entity_id FROM bitrix_sync.entity_external_mapping
|
||||
WHERE external_system='bitrix24' AND external_entity_type='contact'
|
||||
AND external_id=:external_id AND status='active'
|
||||
"""
|
||||
),
|
||||
{"external_id": external_id},
|
||||
)
|
||||
).scalar_one_or_none()
|
||||
|
||||
async def apply_crm_profile(
|
||||
self,
|
||||
user_id: uuid.UUID,
|
||||
external_id: str,
|
||||
*,
|
||||
full_name: str | None,
|
||||
citizenship: str | None,
|
||||
email: str | None,
|
||||
source_updated_at: datetime | None,
|
||||
source: str,
|
||||
) -> None:
|
||||
async with self.engine.begin() as connection:
|
||||
await connection.execute(text("SET LOCAL han.sync_suppress='true'"))
|
||||
await connection.execute(
|
||||
text(
|
||||
"""
|
||||
UPDATE han_app.client_profiles
|
||||
SET full_name=:full_name,citizenship=:citizenship,email=:email,updated_at=now()
|
||||
WHERE user_id=:user_id AND record_status='A'
|
||||
"""
|
||||
),
|
||||
{
|
||||
"user_id": user_id,
|
||||
"full_name": full_name,
|
||||
"citizenship": citizenship,
|
||||
"email": email,
|
||||
},
|
||||
)
|
||||
await connection.execute(
|
||||
text(
|
||||
"""
|
||||
INSERT INTO bitrix_sync.contact_snapshots
|
||||
(id,mapping_id,user_id,external_id,full_name_hash,email_hash,
|
||||
citizenship_hash,source_updated_at,last_applied_source,
|
||||
last_webhook_received_at,created_at,updated_at)
|
||||
SELECT gen_random_uuid(),m.id,:user_id,:external_id,
|
||||
encode(digest(coalesce(:full_name,''),'sha256'),'hex'),
|
||||
encode(digest(coalesce(:email,''),'sha256'),'hex'),
|
||||
encode(digest(coalesce(:citizenship,''),'sha256'),'hex'),
|
||||
:source_updated_at,:source,
|
||||
CASE WHEN :source='webhook' THEN now() END,now(),now()
|
||||
FROM bitrix_sync.entity_external_mapping m
|
||||
WHERE m.entity_id=:user_id AND m.external_id=:external_id AND m.status='active'
|
||||
ON CONFLICT (mapping_id) DO UPDATE
|
||||
SET full_name_hash=excluded.full_name_hash,email_hash=excluded.email_hash,
|
||||
citizenship_hash=excluded.citizenship_hash,
|
||||
source_updated_at=excluded.source_updated_at,
|
||||
last_applied_source=excluded.last_applied_source,
|
||||
last_webhook_received_at=coalesce(
|
||||
excluded.last_webhook_received_at,
|
||||
bitrix_sync.contact_snapshots.last_webhook_received_at),
|
||||
updated_at=now()
|
||||
"""
|
||||
),
|
||||
{
|
||||
"user_id": user_id,
|
||||
"external_id": external_id,
|
||||
"full_name": full_name,
|
||||
"citizenship": citizenship,
|
||||
"email": email,
|
||||
"source_updated_at": source_updated_at,
|
||||
"source": source,
|
||||
},
|
||||
)
|
||||
|
||||
async def complete_webhook(self, item: LeasedWebhook) -> bool:
|
||||
async with self.engine.begin() as connection:
|
||||
result = await connection.execute(
|
||||
text(
|
||||
"""
|
||||
UPDATE bitrix_sync.webhook_inbox
|
||||
SET status='processed',processed_at=now(),locked_by=NULL,
|
||||
locked_until=NULL,lease_token=NULL
|
||||
WHERE id=:id AND status='processing' AND lease_token=:lease_token
|
||||
"""
|
||||
),
|
||||
{"id": item.id, "lease_token": item.lease_token},
|
||||
)
|
||||
return result.rowcount == 1
|
||||
|
||||
async def retry_webhook(
|
||||
self, item: LeasedWebhook, safe_code: str, delay_seconds: float
|
||||
) -> bool:
|
||||
async with self.engine.begin() as connection:
|
||||
result = await connection.execute(
|
||||
text(
|
||||
"""
|
||||
UPDATE bitrix_sync.webhook_inbox
|
||||
SET status='retry_wait',attempt_count=attempt_count+1,
|
||||
next_attempt_at=now()+make_interval(secs=>:delay),
|
||||
safe_error_code=:code,locked_by=NULL,locked_until=NULL,lease_token=NULL
|
||||
WHERE id=:id AND status='processing' AND lease_token=:lease_token
|
||||
"""
|
||||
),
|
||||
{
|
||||
"id": item.id,
|
||||
"lease_token": item.lease_token,
|
||||
"code": safe_code[:64],
|
||||
"delay": delay_seconds,
|
||||
},
|
||||
)
|
||||
return result.rowcount == 1
|
||||
|
||||
async def pending_rebind_ids(self, limit: int) -> list[uuid.UUID]:
|
||||
async with self.engine.connect() as connection:
|
||||
rows = await connection.execute(
|
||||
text(
|
||||
"""
|
||||
SELECT id FROM bitrix_sync.rebind_requests
|
||||
WHERE status IN ('pending','retry_wait')
|
||||
ORDER BY requested_at LIMIT :limit
|
||||
"""
|
||||
),
|
||||
{"limit": limit},
|
||||
)
|
||||
return list(rows.scalars())
|
||||
|
||||
async def reserve_limiter_token(self, refill_per_second: float, burst: int) -> float:
|
||||
async with self.engine.begin() as connection:
|
||||
row = (
|
||||
await connection.execute(
|
||||
text(
|
||||
"""
|
||||
SELECT tokens,capacity,refill_per_second,
|
||||
extract(epoch FROM now()-updated_at) elapsed,
|
||||
greatest(0,extract(epoch FROM blocked_until-now())) blocked
|
||||
FROM bitrix_sync.limiter_coordination
|
||||
WHERE limiter_key='bitrix24:portal'
|
||||
FOR UPDATE
|
||||
"""
|
||||
)
|
||||
)
|
||||
).mappings().first()
|
||||
if row is None:
|
||||
await connection.execute(
|
||||
text(
|
||||
"""
|
||||
INSERT INTO bitrix_sync.limiter_coordination
|
||||
(limiter_key,tokens,capacity,refill_per_second,updated_at,fencing_token)
|
||||
VALUES ('bitrix24:portal',:tokens,:capacity,:refill,now(),1)
|
||||
"""
|
||||
),
|
||||
{"tokens": max(0, burst - 1), "capacity": burst, "refill": refill_per_second},
|
||||
)
|
||||
return 0
|
||||
blocked = float(row["blocked"] or 0)
|
||||
tokens = min(
|
||||
float(burst),
|
||||
float(row["tokens"]) + float(row["elapsed"] or 0) * refill_per_second,
|
||||
)
|
||||
if blocked > 0:
|
||||
delay = blocked
|
||||
elif tokens >= 1:
|
||||
tokens -= 1
|
||||
delay = 0
|
||||
else:
|
||||
delay = (1 - tokens) / refill_per_second
|
||||
await connection.execute(
|
||||
text(
|
||||
"""
|
||||
UPDATE bitrix_sync.limiter_coordination
|
||||
SET tokens=:tokens,capacity=:capacity,refill_per_second=:refill,
|
||||
updated_at=now(),fencing_token=fencing_token+1
|
||||
WHERE limiter_key='bitrix24:portal'
|
||||
"""
|
||||
),
|
||||
{
|
||||
"tokens": tokens,
|
||||
"capacity": burst,
|
||||
"refill": refill_per_second,
|
||||
},
|
||||
)
|
||||
return delay
|
||||
|
||||
async def status(self) -> dict[str, Any]:
|
||||
queries = {
|
||||
"queue": "SELECT status, count(*) count FROM han_app.sync_queue GROUP BY status",
|
||||
"workflows": (
|
||||
"SELECT state, count(*) count FROM bitrix_sync.workflow_instances GROUP BY state"
|
||||
),
|
||||
"commands": (
|
||||
"SELECT status, count(*) count "
|
||||
"FROM bitrix_sync.crm_commands GROUP BY status"
|
||||
),
|
||||
"webhook_lag_seconds": (
|
||||
"SELECT coalesce(extract(epoch from now()-min(received_at)),0) "
|
||||
"FROM bitrix_sync.webhook_inbox WHERE status IN ('received','retry_wait')"
|
||||
),
|
||||
"settings_version": (
|
||||
"SELECT version FROM bitrix_sync.settings_versions "
|
||||
"WHERE active=true AND validation_status='valid' ORDER BY activated_at DESC LIMIT 1"
|
||||
),
|
||||
}
|
||||
output: dict[str, Any] = {}
|
||||
async with self.engine.connect() as connection:
|
||||
for key, sql in queries.items():
|
||||
result = await connection.execute(text(sql))
|
||||
if key in {"queue", "workflows", "commands"}:
|
||||
output[key] = {row.status: row.count for row in result}
|
||||
else:
|
||||
output[key] = result.scalar_one_or_none()
|
||||
output["generated_at"] = datetime.now(UTC).isoformat()
|
||||
return output
|
||||
@@ -0,0 +1,120 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import hmac
|
||||
import ipaddress
|
||||
import re
|
||||
from collections.abc import Mapping
|
||||
from dataclasses import dataclass
|
||||
from urllib.parse import parse_qsl
|
||||
|
||||
from app.config import Settings
|
||||
|
||||
SECRET_PATTERNS = (
|
||||
re.compile(r"(?i)(token|authorization|password|secret)=([^&\s]+)"),
|
||||
re.compile(r"https://[^/\s]+/rest/[0-9]+/[^/\s]+/"),
|
||||
)
|
||||
|
||||
|
||||
def token_matches(received: str | None, current: str, previous: str | None = None) -> bool:
|
||||
candidate = (received or "").encode()
|
||||
current_match = hmac.compare_digest(candidate, current.encode())
|
||||
previous_match = hmac.compare_digest(candidate, (previous or "").encode())
|
||||
return current_match or (previous is not None and previous_match)
|
||||
|
||||
|
||||
def redact(value: object) -> str:
|
||||
text = str(value)
|
||||
for pattern in SECRET_PATTERNS:
|
||||
text = pattern.sub(
|
||||
lambda match: (
|
||||
f"{match.group(1)}=[REDACTED]"
|
||||
if match.lastindex == 2
|
||||
else "https://[REDACTED]/"
|
||||
),
|
||||
text,
|
||||
)
|
||||
if "@" in text or re.search(r"\+7[0-9]{10}", text):
|
||||
return "[PII_REDACTED]"
|
||||
return text[:512]
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class WebhookEvent:
|
||||
receiver_type: str
|
||||
entity_id: str
|
||||
event_type: str
|
||||
source_ip: str
|
||||
|
||||
|
||||
class WebhookValidationError(ValueError):
|
||||
pass
|
||||
|
||||
|
||||
def parse_bounded_form(body: bytes, *, max_fields: int) -> dict[str, str]:
|
||||
try:
|
||||
pairs = parse_qsl(body.decode("utf-8"), keep_blank_values=True, max_num_fields=max_fields)
|
||||
except (UnicodeDecodeError, ValueError) as exc:
|
||||
raise WebhookValidationError("malformed form") from exc
|
||||
if len(pairs) > max_fields:
|
||||
raise WebhookValidationError("too many form fields")
|
||||
data: dict[str, str] = {}
|
||||
for key, value in pairs:
|
||||
if len(key) > 128 or len(value) > 512:
|
||||
raise WebhookValidationError("form field too long")
|
||||
data[key] = value
|
||||
return data
|
||||
|
||||
|
||||
def validate_webhook(
|
||||
receiver: str,
|
||||
query: Mapping[str, str],
|
||||
form: Mapping[str, str],
|
||||
source_ip: str,
|
||||
settings: Settings,
|
||||
*,
|
||||
alert_entity_type_id: int | None,
|
||||
) -> WebhookEvent:
|
||||
try:
|
||||
ip = ipaddress.ip_address(source_ip)
|
||||
except ValueError as exc:
|
||||
raise WebhookValidationError("invalid source address") from exc
|
||||
if not any(ip in network for network in settings.allowed_networks):
|
||||
raise PermissionError("source_ip")
|
||||
|
||||
configured = (
|
||||
settings.contact_receiver_token if receiver == "contact" else settings.alert_receiver_token
|
||||
)
|
||||
previous = (
|
||||
settings.contact_receiver_previous_token
|
||||
if receiver == "contact"
|
||||
else settings.alert_receiver_previous_token
|
||||
)
|
||||
if not configured or not token_matches(
|
||||
query.get("token"),
|
||||
configured.get_secret_value(),
|
||||
previous.get_secret_value() if previous else None,
|
||||
):
|
||||
raise PermissionError("token")
|
||||
if form.get("auth[domain]", "").lower() != str(settings.portal_host).lower():
|
||||
raise WebhookValidationError("portal mismatch")
|
||||
if form.get("auth[member_id]") != settings.portal_member_id:
|
||||
raise WebhookValidationError("member mismatch")
|
||||
if form.get("document_id[0]") != "crm":
|
||||
raise WebhookValidationError("invalid document module")
|
||||
|
||||
document_type = form.get("document_id[1]")
|
||||
document_id = form.get("document_id[2]", "")
|
||||
if receiver == "contact":
|
||||
match = re.fullmatch(r"CONTACT_([1-9][0-9]*)", document_id)
|
||||
if document_type != "CCrmDocumentContact" or not match:
|
||||
raise WebhookValidationError("invalid contact document")
|
||||
else:
|
||||
match = re.fullmatch(r"DYNAMIC_([1-9][0-9]*)_([1-9][0-9]*)", document_id)
|
||||
if not match or not document_type or "Dynamic" not in document_type:
|
||||
raise WebhookValidationError("invalid alert document")
|
||||
if alert_entity_type_id is None or int(match.group(1)) != alert_entity_type_id:
|
||||
raise WebhookValidationError("alert entity type mismatch")
|
||||
entity_id = match.group(match.lastindex or 1)
|
||||
if query.get("ID") != entity_id:
|
||||
raise WebhookValidationError("query/document ID mismatch")
|
||||
return WebhookEvent(receiver, entity_id, f"{receiver}.changed", source_ip)
|
||||
@@ -0,0 +1,78 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import signal
|
||||
import socket
|
||||
import uuid
|
||||
|
||||
from app.config import load_settings
|
||||
from app.crm import CrmClient
|
||||
from app.domain import full_jitter_delay
|
||||
from app.engine import RetryableWorkflow, WorkflowEngine
|
||||
from app.repository import Repository
|
||||
|
||||
|
||||
async def worker_main() -> None:
|
||||
settings = load_settings()
|
||||
if not settings.enabled:
|
||||
return
|
||||
assert settings.database_url and settings.crm_rest_webhook_url and settings.portal_host
|
||||
repository = Repository(settings.database_url.get_secret_value(), settings.db_pool_size)
|
||||
crm = CrmClient(
|
||||
settings.crm_rest_webhook_url.get_secret_value(),
|
||||
settings.portal_host,
|
||||
settings.http_timeout_sec,
|
||||
)
|
||||
engine = WorkflowEngine(repository, crm, settings)
|
||||
stop = asyncio.Event()
|
||||
loop = asyncio.get_running_loop()
|
||||
for event in (signal.SIGINT, signal.SIGTERM):
|
||||
try:
|
||||
loop.add_signal_handler(event, stop.set)
|
||||
except NotImplementedError:
|
||||
pass
|
||||
worker_id = f"{socket.gethostname()}:{uuid.uuid4()}"
|
||||
try:
|
||||
while not stop.is_set():
|
||||
tasks = await repository.claim_tasks(
|
||||
worker_id, settings.claim_size, settings.lease_seconds
|
||||
)
|
||||
webhooks = await repository.claim_webhooks(
|
||||
worker_id, settings.claim_size, settings.lease_seconds
|
||||
)
|
||||
rebind_ids = await repository.pending_rebind_ids(settings.claim_size)
|
||||
if not tasks and not webhooks and not rebind_ids:
|
||||
try:
|
||||
await asyncio.wait_for(stop.wait(), timeout=1)
|
||||
except TimeoutError:
|
||||
continue
|
||||
for task in tasks:
|
||||
if stop.is_set():
|
||||
break
|
||||
await engine.process(task)
|
||||
for item in webhooks:
|
||||
if stop.is_set():
|
||||
break
|
||||
try:
|
||||
await engine.process_webhook(item)
|
||||
except RetryableWorkflow as exc:
|
||||
delay = exc.retry_after or full_jitter_delay(
|
||||
item.attempt_count + 1,
|
||||
settings.retry_base_seconds,
|
||||
settings.retry_max_seconds,
|
||||
)
|
||||
await repository.retry_webhook(item, exc.code, delay)
|
||||
for request_id in rebind_ids:
|
||||
if stop.is_set():
|
||||
break
|
||||
try:
|
||||
await engine.process_rebind(request_id)
|
||||
except RetryableWorkflow:
|
||||
continue
|
||||
finally:
|
||||
await crm.close()
|
||||
await repository.close()
|
||||
|
||||
|
||||
def run() -> None:
|
||||
asyncio.run(worker_main())
|
||||
@@ -0,0 +1,83 @@
|
||||
services:
|
||||
bitrix-sync:
|
||||
build: .
|
||||
image: han-bitrix-sync:${BITRIX_SYNC_IMAGE_TAG:-local}
|
||||
command: ["han-bitrix-sync-api"]
|
||||
user: "10001:10001"
|
||||
read_only: true
|
||||
security_opt: ["no-new-privileges:true"]
|
||||
cap_drop: ["ALL"]
|
||||
tmpfs: ["/tmp:rw,noexec,nosuid,nodev,size=16m"]
|
||||
expose: ["8080"]
|
||||
environment: &sync_environment
|
||||
BITRIX_SYNC_ENABLED: ${BITRIX_SYNC_ENABLED:-false}
|
||||
BITRIX_SYNC_MODE: ${BITRIX_SYNC_MODE:-disabled}
|
||||
BITRIX_SYNC_DATABASE_URL_FILE: /run/secrets/bitrix_sync_database_url
|
||||
BITRIX_SYNC_CRM_REST_WEBHOOK_URL_FILE: /run/secrets/bitrix_sync_crm_url
|
||||
BITRIX_SYNC_CONTACT_RECEIVER_TOKEN_FILE: /run/secrets/bitrix_sync_contact_token
|
||||
BITRIX_SYNC_ALERT_RECEIVER_TOKEN_FILE: /run/secrets/bitrix_sync_alert_token
|
||||
BITRIX_SYNC_SERVICE_TOKEN_FILE: /run/secrets/bitrix_sync_service_token
|
||||
BITRIX_SYNC_PORTAL_HOST: ${BITRIX_SYNC_PORTAL_HOST:-}
|
||||
BITRIX_SYNC_PORTAL_MEMBER_ID: ${BITRIX_SYNC_PORTAL_MEMBER_ID:-}
|
||||
BITRIX_SYNC_PUBLIC_BASE_URL: ${BITRIX_SYNC_PUBLIC_BASE_URL:-}
|
||||
BITRIX_SYNC_CONTACT_USER_ID_FIELD: ${BITRIX_SYNC_CONTACT_USER_ID_FIELD:-}
|
||||
BITRIX_SYNC_CONTACT_REGISTERED_FIELD: ${BITRIX_SYNC_CONTACT_REGISTERED_FIELD:-}
|
||||
BITRIX_SYNC_CONTACT_CITIZENSHIP_FIELD: ${BITRIX_SYNC_CONTACT_CITIZENSHIP_FIELD:-}
|
||||
BITRIX_SYNC_WEBHOOK_ALLOWED_CIDRS: ${BITRIX_WEBHOOK_ALLOWED_CIDRS:-}
|
||||
volumes: &sync_secrets
|
||||
- /run/han-chat/secrets/bitrix-sync/database-url:/run/secrets/bitrix_sync_database_url:ro
|
||||
- /run/han-chat/secrets/bitrix-sync/crm-rest-webhook-url:/run/secrets/bitrix_sync_crm_url:ro
|
||||
- /run/han-chat/secrets/bitrix-sync/contact-receiver-token:/run/secrets/bitrix_sync_contact_token:ro
|
||||
- /run/han-chat/secrets/bitrix-sync/alert-receiver-token:/run/secrets/bitrix_sync_alert_token:ro
|
||||
- /run/han-chat/secrets/bitrix-sync/service-token:/run/secrets/bitrix_sync_service_token:ro
|
||||
networks: [backend, egress, observability]
|
||||
pids_limit: 128
|
||||
mem_limit: 256m
|
||||
cpus: 0.50
|
||||
restart: unless-stopped
|
||||
healthcheck:
|
||||
test:
|
||||
["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health/live',timeout=2)"]
|
||||
interval: 30s
|
||||
timeout: 3s
|
||||
retries: 3
|
||||
|
||||
bitrix-sync-worker:
|
||||
image: han-bitrix-sync:${BITRIX_SYNC_IMAGE_TAG:-local}
|
||||
command: ["han-bitrix-sync-worker"]
|
||||
user: "10001:10001"
|
||||
read_only: true
|
||||
security_opt: ["no-new-privileges:true"]
|
||||
cap_drop: ["ALL"]
|
||||
tmpfs: ["/tmp:rw,noexec,nosuid,nodev,size=16m"]
|
||||
environment: *sync_environment
|
||||
volumes: *sync_secrets
|
||||
networks: [egress, observability]
|
||||
pids_limit: 128
|
||||
mem_limit: 256m
|
||||
cpus: 0.75
|
||||
restart: unless-stopped
|
||||
|
||||
bitrix-sync-reconciliation:
|
||||
image: han-bitrix-sync:${BITRIX_SYNC_IMAGE_TAG:-local}
|
||||
command: ["han-bitrix-sync-reconciliation"]
|
||||
user: "10001:10001"
|
||||
read_only: true
|
||||
security_opt: ["no-new-privileges:true"]
|
||||
cap_drop: ["ALL"]
|
||||
tmpfs: ["/tmp:rw,noexec,nosuid,nodev,size=16m"]
|
||||
environment: *sync_environment
|
||||
volumes: *sync_secrets
|
||||
networks: [egress, observability]
|
||||
pids_limit: 128
|
||||
mem_limit: 192m
|
||||
cpus: 0.50
|
||||
restart: "no"
|
||||
|
||||
networks:
|
||||
backend:
|
||||
external: true
|
||||
egress:
|
||||
external: true
|
||||
observability:
|
||||
external: true
|
||||
@@ -0,0 +1,121 @@
|
||||
openapi: 3.1.0
|
||||
info:
|
||||
title: HAN Bitrix Sync
|
||||
version: 0.1.0
|
||||
paths:
|
||||
/health/live:
|
||||
get:
|
||||
operationId: healthLive
|
||||
responses:
|
||||
"200":
|
||||
description: Process is alive
|
||||
/health/ready:
|
||||
get:
|
||||
operationId: healthReady
|
||||
responses:
|
||||
"200":
|
||||
description: Full-mode configuration and database are ready
|
||||
"503":
|
||||
description: Disabled or a core dependency is not ready
|
||||
/internal/sync/v1/status:
|
||||
get:
|
||||
operationId: syncStatus
|
||||
security:
|
||||
- bearerAuth: []
|
||||
responses:
|
||||
"200":
|
||||
description: Low-cardinality operational status without PII
|
||||
"401":
|
||||
description: Missing or invalid service token
|
||||
/bitrix/sync/webhook/contact:
|
||||
post:
|
||||
operationId: receiveContactRobot
|
||||
parameters:
|
||||
- $ref: "#/components/parameters/ReceiverToken"
|
||||
- $ref: "#/components/parameters/EntityId"
|
||||
requestBody:
|
||||
required: true
|
||||
content:
|
||||
application/x-www-form-urlencoded:
|
||||
schema:
|
||||
$ref: "#/components/schemas/RobotForm"
|
||||
responses:
|
||||
"202":
|
||||
description: Event durably stored
|
||||
"400":
|
||||
description: Malformed robot contract
|
||||
"403":
|
||||
description: Receiver authentication rejected
|
||||
"413":
|
||||
description: Body too large
|
||||
"503":
|
||||
description: Sync is disabled
|
||||
/bitrix/sync/webhook/alert:
|
||||
post:
|
||||
operationId: receiveAlertRobot
|
||||
parameters:
|
||||
- $ref: "#/components/parameters/ReceiverToken"
|
||||
- $ref: "#/components/parameters/EntityId"
|
||||
requestBody:
|
||||
required: true
|
||||
content:
|
||||
application/x-www-form-urlencoded:
|
||||
schema:
|
||||
$ref: "#/components/schemas/RobotForm"
|
||||
responses:
|
||||
"202":
|
||||
description: Event durably stored
|
||||
"400":
|
||||
description: Malformed robot contract
|
||||
"403":
|
||||
description: Receiver authentication rejected
|
||||
"413":
|
||||
description: Body too large
|
||||
"503":
|
||||
description: Sync is disabled
|
||||
components:
|
||||
securitySchemes:
|
||||
bearerAuth:
|
||||
type: http
|
||||
scheme: bearer
|
||||
parameters:
|
||||
ReceiverToken:
|
||||
name: token
|
||||
in: query
|
||||
required: true
|
||||
schema:
|
||||
type: string
|
||||
maxLength: 256
|
||||
description: Secret receiver token; MUST be excluded from logs and traces.
|
||||
EntityId:
|
||||
name: ID
|
||||
in: query
|
||||
required: true
|
||||
schema:
|
||||
type: string
|
||||
pattern: "^[1-9][0-9]{0,19}$"
|
||||
schemas:
|
||||
RobotForm:
|
||||
type: object
|
||||
additionalProperties: false
|
||||
required:
|
||||
- document_id[0]
|
||||
- document_id[1]
|
||||
- document_id[2]
|
||||
- auth[domain]
|
||||
- auth[member_id]
|
||||
properties:
|
||||
document_id[0]:
|
||||
type: string
|
||||
document_id[1]:
|
||||
type: string
|
||||
document_id[2]:
|
||||
type: string
|
||||
auth[domain]:
|
||||
type: string
|
||||
auth[member_id]:
|
||||
type: string
|
||||
auth[client_endpoint]:
|
||||
type: string
|
||||
auth[server_endpoint]:
|
||||
type: string
|
||||
@@ -0,0 +1,50 @@
|
||||
[project]
|
||||
name = "han-bitrix-sync"
|
||||
version = "0.1.0"
|
||||
description = "Durable HAN App to Bitrix24 Contact synchronization"
|
||||
requires-python = ">=3.12"
|
||||
dependencies = [
|
||||
"alembic>=1.16,<2",
|
||||
"asyncpg>=0.30,<1",
|
||||
"fastapi>=0.116,<1",
|
||||
"httpx>=0.28,<1",
|
||||
"pydantic-settings>=2.10,<3",
|
||||
"python-multipart>=0.0.20,<1",
|
||||
"sqlalchemy[asyncio]>=2.0.41,<3",
|
||||
"structlog>=25,<26",
|
||||
"uvicorn[standard]>=0.35,<1",
|
||||
]
|
||||
|
||||
[project.optional-dependencies]
|
||||
dev = [
|
||||
"pytest>=8.4,<9",
|
||||
"pytest-asyncio>=1.0,<2",
|
||||
"ruff>=0.12,<1",
|
||||
]
|
||||
|
||||
[project.scripts]
|
||||
han-bitrix-sync-api = "app.main:run"
|
||||
han-bitrix-sync-worker = "app.worker:run"
|
||||
han-bitrix-sync-reconciliation = "app.reconciliation:run"
|
||||
|
||||
[build-system]
|
||||
requires = ["hatchling"]
|
||||
build-backend = "hatchling.build"
|
||||
|
||||
[tool.hatch.build.targets.wheel]
|
||||
packages = ["app"]
|
||||
|
||||
[tool.pytest.ini_options]
|
||||
asyncio_mode = "auto"
|
||||
testpaths = ["tests"]
|
||||
|
||||
[tool.ruff]
|
||||
target-version = "py312"
|
||||
line-length = 100
|
||||
|
||||
[tool.ruff.lint]
|
||||
select = ["E", "F", "I", "UP", "B", "ASYNC", "S"]
|
||||
ignore = ["S101"]
|
||||
|
||||
[tool.ruff.lint.per-file-ignores]
|
||||
"tests/**" = ["S106", "S311"]
|
||||
@@ -0,0 +1,25 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
|
||||
from app.config import Settings
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def full_settings() -> Settings:
|
||||
return Settings(
|
||||
enabled=True,
|
||||
mode="full",
|
||||
database_url="postgresql+asyncpg://user:pass@db/han",
|
||||
crm_rest_webhook_url="https://portal.example/rest/1/credential/",
|
||||
contact_receiver_token="contact-token-value-32-characters",
|
||||
alert_receiver_token="alert-token-value-32-characters---",
|
||||
service_token="service-token-value-32-characters-",
|
||||
portal_host="portal.example",
|
||||
portal_member_id="member_12345678",
|
||||
public_base_url="https://sync.example",
|
||||
contact_user_id_field="UF_CRM_100",
|
||||
contact_registered_field="UF_CRM_101",
|
||||
contact_citizenship_field="UF_CRM_102",
|
||||
webhook_allowed_cidrs="203.0.113.0/24",
|
||||
)
|
||||
@@ -0,0 +1,77 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import ast
|
||||
import re
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
from pydantic import ValidationError
|
||||
|
||||
from app.config import Settings
|
||||
|
||||
|
||||
def test_disabled_mode_needs_no_secrets() -> None:
|
||||
settings = Settings(enabled=False, mode="disabled")
|
||||
assert settings.enabled is False
|
||||
|
||||
|
||||
def test_full_mode_rejects_portal_host_mismatch() -> None:
|
||||
with pytest.raises(ValidationError, match="approved portal host"):
|
||||
Settings(
|
||||
enabled=True,
|
||||
mode="full",
|
||||
database_url="postgresql+asyncpg://u:p@db/han",
|
||||
crm_rest_webhook_url="https://evil.example/rest/1/token/",
|
||||
contact_receiver_token="contact",
|
||||
alert_receiver_token="alert",
|
||||
service_token="service",
|
||||
portal_host="portal.example",
|
||||
portal_member_id="member_12345678",
|
||||
public_base_url="https://sync.example",
|
||||
contact_user_id_field="UF_CRM_1",
|
||||
contact_registered_field="UF_CRM_2",
|
||||
contact_citizenship_field="UF_CRM_3",
|
||||
webhook_allowed_cidrs="203.0.113.0/24",
|
||||
)
|
||||
|
||||
|
||||
def test_rest_field_conversion_is_deterministic() -> None:
|
||||
assert Settings.rest_field_name("UF_CRM_1778692456") == "ufCrm_1778692456"
|
||||
with pytest.raises(ValueError):
|
||||
Settings.rest_field_name("uf_crm_1")
|
||||
|
||||
|
||||
def test_alembic_chain_preserves_legacy_baseline() -> None:
|
||||
versions = Path(__file__).parents[1] / "alembic" / "versions"
|
||||
revisions: dict[str, str | None] = {}
|
||||
|
||||
for migration in versions.glob("*.py"):
|
||||
assignments = {
|
||||
node.target.id: node.value.value
|
||||
for node in ast.parse(migration.read_text(encoding="utf-8")).body
|
||||
if isinstance(node, ast.AnnAssign)
|
||||
and isinstance(node.target, ast.Name)
|
||||
and node.target.id in {"revision", "down_revision"}
|
||||
and isinstance(node.value, ast.Constant)
|
||||
}
|
||||
revisions[assignments["revision"]] = assignments["down_revision"]
|
||||
|
||||
assert revisions == {
|
||||
"0001_sync_baseline": None,
|
||||
"0001_bitrix_sync_full": "0001_sync_baseline",
|
||||
"0002_app_queue_contract": "0001_bitrix_sync_full",
|
||||
}
|
||||
|
||||
|
||||
def test_app_queue_migration_does_not_revoke_foreign_schema_privileges() -> None:
|
||||
migration = (
|
||||
Path(__file__).parents[1]
|
||||
/ "alembic"
|
||||
/ "versions"
|
||||
/ "0002_app_queue_contract.py"
|
||||
)
|
||||
source = migration.read_text(encoding="utf-8")
|
||||
|
||||
assert "FROM han_app.entity_external_mapping" in source
|
||||
assert "REVOKE" not in source
|
||||
assert re.search(r'"[^"]+"\s*:', source) is None
|
||||
@@ -0,0 +1,48 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
|
||||
from app.crm import CrmClient, CrmOutcome
|
||||
|
||||
|
||||
def make_client(handler) -> CrmClient:
|
||||
client = CrmClient.__new__(CrmClient)
|
||||
client._base_url = "https://portal.example/rest/1/token/"
|
||||
client._host = "portal.example"
|
||||
client._client = httpx.AsyncClient(
|
||||
transport=httpx.MockTransport(handler), follow_redirects=False
|
||||
)
|
||||
return client
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_crm_success_and_no_redirect() -> None:
|
||||
client = make_client(
|
||||
lambda request: httpx.Response(200, json={"result": {"ID": "42"}}, request=request)
|
||||
)
|
||||
result = await client.call("crm.contact.get", {"id": "42"}, mutating=False)
|
||||
assert result.outcome == CrmOutcome.SUCCEEDED
|
||||
assert result.result["ID"] == "42"
|
||||
await client.close()
|
||||
|
||||
redirecting = make_client(
|
||||
lambda request: httpx.Response(
|
||||
302, headers={"Location": "https://evil.example/"}, request=request
|
||||
)
|
||||
)
|
||||
result = await redirecting.call("crm.contact.get", {"id": "42"}, mutating=False)
|
||||
assert result.outcome == CrmOutcome.PERMANENT
|
||||
assert result.error_code == "crm_redirect_rejected"
|
||||
await redirecting.close()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_mutating_timeout_is_uncertain() -> None:
|
||||
def timeout(request):
|
||||
raise httpx.ReadTimeout("timed out", request=request)
|
||||
|
||||
client = make_client(timeout)
|
||||
result = await client.call("crm.contact.add", {"fields": {}}, mutating=True)
|
||||
assert result.outcome == CrmOutcome.UNCERTAIN
|
||||
await client.close()
|
||||
@@ -0,0 +1,57 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import random
|
||||
from datetime import UTC, datetime, timedelta
|
||||
|
||||
import pytest
|
||||
|
||||
from app.domain import (
|
||||
ContactCandidate,
|
||||
WorkflowState,
|
||||
assert_transition,
|
||||
choose_newest,
|
||||
full_jitter_delay,
|
||||
select_email,
|
||||
validate_phone,
|
||||
)
|
||||
from app.mapping import CitizenshipDictionary, UnknownCitizenship
|
||||
|
||||
|
||||
def test_contact_choice_has_numeric_id_tie_breaker() -> None:
|
||||
created = datetime(2026, 8, 6, tzinfo=UTC)
|
||||
selected = choose_newest(
|
||||
[
|
||||
ContactCandidate("9", created, None),
|
||||
ContactCandidate("10", created, None),
|
||||
]
|
||||
)
|
||||
assert selected and selected.b24_id == "10"
|
||||
|
||||
|
||||
def test_phone_email_and_citizenship_mapping() -> None:
|
||||
assert validate_phone("+79001234567") == "+79001234567"
|
||||
with pytest.raises(ValueError):
|
||||
validate_phone("8 900 123-45-67")
|
||||
assert (
|
||||
select_email(
|
||||
[
|
||||
{"VALUE": "home@example.test", "VALUE_TYPE": "HOME"},
|
||||
{"VALUE": "work@example.test", "VALUE_TYPE": "WORK"},
|
||||
]
|
||||
)
|
||||
== "work@example.test"
|
||||
)
|
||||
dictionary = CitizenshipDictionary(ttl_seconds=60)
|
||||
now = datetime(2026, 8, 6, tzinfo=UTC)
|
||||
dictionary.load([{"ID": "7", "VALUE": "Казахстан"}], now)
|
||||
assert dictionary.resolve("7", now + timedelta(seconds=30)) == "Казахстан"
|
||||
with pytest.raises(UnknownCitizenship):
|
||||
dictionary.resolve("8", now)
|
||||
|
||||
|
||||
def test_state_machine_and_retry_bounds() -> None:
|
||||
assert_transition(WorkflowState.CREATED, WorkflowState.RUNNING)
|
||||
with pytest.raises(ValueError):
|
||||
assert_transition(WorkflowState.SUCCEEDED, WorkflowState.RUNNING)
|
||||
delay = full_jitter_delay(4, 1, 5, rng=random.Random(1))
|
||||
assert 0 <= delay <= 5
|
||||
@@ -0,0 +1,72 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import uuid
|
||||
from contextlib import asynccontextmanager
|
||||
|
||||
import pytest
|
||||
|
||||
from app.crm import CrmOutcome, CrmResult
|
||||
from app.engine import WorkflowEngine
|
||||
from app.repository import Profile
|
||||
|
||||
|
||||
class Result:
|
||||
rowcount = 1
|
||||
|
||||
|
||||
class Connection:
|
||||
async def execute(self, statement, params=None):
|
||||
return Result()
|
||||
|
||||
|
||||
class FakeRepository:
|
||||
def __init__(self) -> None:
|
||||
self.mapping = None
|
||||
self.statements: list[str] = []
|
||||
|
||||
@asynccontextmanager
|
||||
async def transaction(self):
|
||||
yield Connection()
|
||||
|
||||
async def active_mapping(self, user_id):
|
||||
return self.mapping
|
||||
|
||||
async def reserve_limiter_token(self, refill_per_second, burst):
|
||||
return 0
|
||||
|
||||
|
||||
class FakeCrm:
|
||||
def __init__(self, user_id: uuid.UUID) -> None:
|
||||
self.user_id = user_id
|
||||
self.calls: list[tuple[str, dict]] = []
|
||||
|
||||
async def call(self, method, params, *, mutating):
|
||||
self.calls.append((method, params))
|
||||
if method == "crm.duplicate.findbycomm":
|
||||
return CrmResult(CrmOutcome.SUCCEEDED, {"CONTACT": ["9", "10"]})
|
||||
if method == "crm.contact.get":
|
||||
contact_id = str(params["id"])
|
||||
return CrmResult(
|
||||
CrmOutcome.SUCCEEDED,
|
||||
{
|
||||
"ID": contact_id,
|
||||
"CREATED_TIME": "2026-08-06T10:00:00Z",
|
||||
"UF_CRM_100": None,
|
||||
},
|
||||
)
|
||||
return CrmResult(CrmOutcome.SUCCEEDED, True)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_multiple_contacts_choose_numeric_newest(full_settings) -> None:
|
||||
user_id = uuid.uuid4()
|
||||
repository = FakeRepository()
|
||||
crm = FakeCrm(user_id)
|
||||
engine = WorkflowEngine(repository, crm, full_settings)
|
||||
await engine._map_or_create(
|
||||
uuid.uuid4(),
|
||||
Profile(user_id=user_id, phone="+79001234567", identity_status="A", profile_status="A"),
|
||||
)
|
||||
updates = [params for method, params in crm.calls if method == "crm.contact.update"]
|
||||
assert updates[0]["id"] == "10"
|
||||
assert not any(method == "crm.contact.add" for method, _ in crm.calls)
|
||||
@@ -0,0 +1,62 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
|
||||
from app.security import (
|
||||
WebhookValidationError,
|
||||
parse_bounded_form,
|
||||
redact,
|
||||
token_matches,
|
||||
validate_webhook,
|
||||
)
|
||||
|
||||
|
||||
def test_contact_webhook_contract(full_settings) -> None:
|
||||
event = validate_webhook(
|
||||
"contact",
|
||||
{"token": "contact-token-value-32-characters", "ID": "42"},
|
||||
{
|
||||
"document_id[0]": "crm",
|
||||
"document_id[1]": "CCrmDocumentContact",
|
||||
"document_id[2]": "CONTACT_42",
|
||||
"auth[domain]": "portal.example",
|
||||
"auth[member_id]": "member_12345678",
|
||||
"auth[client_endpoint]": "https://attacker.invalid/rest/",
|
||||
},
|
||||
"203.0.113.10",
|
||||
full_settings,
|
||||
alert_entity_type_id=None,
|
||||
)
|
||||
assert event.entity_id == "42"
|
||||
|
||||
|
||||
def test_webhook_rejects_document_query_mismatch(full_settings) -> None:
|
||||
with pytest.raises(WebhookValidationError, match="mismatch"):
|
||||
validate_webhook(
|
||||
"contact",
|
||||
{"token": "contact-token-value-32-characters", "ID": "41"},
|
||||
{
|
||||
"document_id[0]": "crm",
|
||||
"document_id[1]": "CCrmDocumentContact",
|
||||
"document_id[2]": "CONTACT_42",
|
||||
"auth[domain]": "portal.example",
|
||||
"auth[member_id]": "member_12345678",
|
||||
},
|
||||
"203.0.113.10",
|
||||
full_settings,
|
||||
alert_entity_type_id=None,
|
||||
)
|
||||
|
||||
|
||||
def test_bounded_form_and_constant_time_token_helpers() -> None:
|
||||
assert parse_bounded_form(b"a=1&b=2", max_fields=2) == {"a": "1", "b": "2"}
|
||||
with pytest.raises(WebhookValidationError):
|
||||
parse_bounded_form(b"a=1&b=2&c=3", max_fields=2)
|
||||
assert token_matches("old", "new", "old")
|
||||
assert not token_matches("other", "new", "old")
|
||||
|
||||
|
||||
def test_redaction_removes_pii_and_secrets() -> None:
|
||||
assert "secret-value" not in redact("token=secret-value")
|
||||
assert redact("user@example.test") == "[PII_REDACTED]"
|
||||
assert redact("+79001234567") == "[PII_REDACTED]"
|
||||
Reference in New Issue
Block a user