Проект разделен на два репозитория

This commit is contained in:
mi
2026-08-14 15:42:45 +03:00
parent e06a77ee1d
commit bbef7a30c9
521 changed files with 2597 additions and 2302 deletions
@@ -0,0 +1,16 @@
BITRIX_SYNC_ENABLED=false
BITRIX_SYNC_MODE=disabled
BITRIX_SYNC_DATABASE_URL_FILE=/run/secrets/bitrix_sync_database_url
BITRIX_SYNC_CRM_REST_WEBHOOK_URL_FILE=/run/secrets/bitrix_sync_crm_url
BITRIX_SYNC_CONTACT_RECEIVER_TOKEN_FILE=/run/secrets/bitrix_sync_contact_token
BITRIX_SYNC_ALERT_RECEIVER_TOKEN_FILE=/run/secrets/bitrix_sync_alert_token
BITRIX_SYNC_SERVICE_TOKEN_FILE=/run/secrets/bitrix_sync_service_token
BITRIX_SYNC_PORTAL_HOST=portal.example.bitrix24.ru
BITRIX_SYNC_PORTAL_MEMBER_ID=replace-with-member-id
BITRIX_SYNC_PUBLIC_BASE_URL=https://sync.example.ru
BITRIX_SYNC_CONTACT_USER_ID_FIELD=UF_CRM_100
BITRIX_SYNC_CONTACT_REGISTERED_FIELD=UF_CRM_101
BITRIX_SYNC_CONTACT_CITIZENSHIP_FIELD=UF_CRM_102
BITRIX_SYNC_WEBHOOK_ALLOWED_CIDRS=203.0.113.0/24
BITRIX_SYNC_HTTP_TIMEOUT_SEC=10
BITRIX_SYNC_DB_POOL_SIZE=5
@@ -0,0 +1,18 @@
ARG PYTHON_IMAGE=python:3.12.11-slim
FROM ${PYTHON_IMAGE} AS build
WORKDIR /build
COPY pyproject.toml ./
COPY app ./app
RUN pip install --no-cache-dir --prefix=/install .
FROM ${PYTHON_IMAGE}
ENV PYTHONDONTWRITEBYTECODE=1 PYTHONUNBUFFERED=1 PATH="/opt/venv/bin:${PATH}"
RUN groupadd --gid 10001 han && useradd --uid 10001 --gid 10001 --no-create-home han
COPY --from=build /install /usr/local
COPY --chown=10001:10001 app /srv/app
COPY --chown=10001:10001 alembic /srv/alembic
COPY --chown=10001:10001 alembic.ini openapi.yaml /srv/
WORKDIR /srv
USER 10001:10001
EXPOSE 8080
ENTRYPOINT ["han-bitrix-sync-api"]
@@ -0,0 +1,72 @@
# bitrix-sync
Изолированный Python 3.12 сервис durable-синхронизации Contact между `han_app` и
Битрикс24. Сервис не участвует в Open Lines и не имеет HTTP-зависимости от
`api-backend`.
## Entrypoints
- `han-bitrix-sync-api` — health, internal status и два bounded robot receiver;
- `han-bitrix-sync-worker` — queue/webhook/rebind workflows с lease fencing;
- `han-bitrix-sync-reconciliation` — один advisory-lock incremental run;
- `alembic upgrade head` — отдельная контролируемая миграция, не startup DDL.
Disabled mode требует только `BITRIX_SYNC_ENABLED=false` и
`BITRIX_SYNC_MODE=disabled`, не читает БД и не принимает webhook. Full mode
валидирует весь каталог secret files, portal identity, custom fields, HTTPS host
lock и непустой CIDR allow-list до startup.
## Границы безопасности
- CRM credential URL используется как единый секрет; redirect выключен, TLS
проверяется, REST method выбирается только из закрытого allow-list.
- Receiver принимает только `application/x-www-form-urlencoded` с bounded
content length, числом и длиной полей. Query token сравнивается constant-time.
- nginx должен перезаписывать `X-Real-IP` из TCP peer, проверять CIDR до proxy и
не логировать `$request_uri`, args или body. Контейнер receiver недоступен
напрямую.
- DB хранит только hash CRM-master значений в snapshot; safe command projection
не содержит PII. URL credential, form body и token не логируются.
- Запись CRM-master полей выполняется в одной транзакции после
`SET LOCAL han.sync_suppress='true'`.
## Локальные проверки
Лёгкие проверки, не требующие Docker, сервиса или реального PostgreSQL:
```text
python -m pytest
python -m ruff check app tests
```
PostgreSQL integration и Bitrix contract suites намеренно являются внешними
gates: локальный managed PostgreSQL не поднимается Compose-файлом.
## External gates до `BITRIX_SYNC_ENABLED=true`
1. Применить migrations migration-role и проверить grants runtime-role.
2. На disposable managed PostgreSQL проверить concurrent `SKIP LOCKED`,
lease expiry/fencing, active mapping uniqueness, rebind partial failure,
transaction-local GUC без утечки и crash после CRM success.
3. Подтвердить на целевом портале wire-контракты `duplicate.findbycomm`,
Contact add/get/update, mixed `batch`, custom fields, enum dictionary и
`crm.item.list` с `opened=1`, registration REST field `=1`.
4. Заполнить и активировать валидную `business_alerts` settings version:
entity/category/stage/field IDs и responsible party. Placeholder `null`
запрещает alert receiver.
5. Проверить least-privilege credential negative tests; credential администратора
запрещён.
6. Валидировать nginx exact routes, no-redirect HTTP policy, body/rate limits,
version-controlled CIDR и отсутствие query/body в access/error/traces.
7. Запустить synthetic webhook с реальным robot form contract, затем убедиться,
что durable inbox commit предшествует `202`.
8. Выполнить 10k incremental reconciliation/load gate, webhook-loss recovery,
429/5xx/TLS/DNS/timeout/uncertain-create и restart-at-each-step tests.
9. Проверить container image digest, dependency/SBOM/vulnerability scan и
compose hardening; root Compose ВМ2 подключает этот fragment отдельно.
10. Зафиксировать cutover watermark, отменить только pre-cutover active tasks,
выполнить disabled preflight и затем controlled enablement.
`compose.fragment.yaml` — сервисный фрагмент, не root Compose и не команда
развёртывания. Reconciliation entrypoint выполняет один run; расписание задаёт
root-owned scheduler/deployment layer.
@@ -0,0 +1,30 @@
[alembic]
script_location = alembic
prepend_sys_path = .
sqlalchemy.url = postgresql+asyncpg://unused
[loggers]
keys = root,sqlalchemy,alembic
[handlers]
keys = console
[formatters]
keys = generic
[logger_root]
level = WARN
handlers = console
qualname =
[logger_sqlalchemy]
level = WARN
handlers =
qualname = sqlalchemy.engine
[logger_alembic]
level = INFO
handlers =
qualname = alembic
[handler_console]
class = StreamHandler
args = (sys.stderr,)
level = NOTSET
formatter = generic
[formatter_generic]
format = %(levelname)-5.5s [%(name)s] %(message)s
@@ -0,0 +1,58 @@
from __future__ import annotations
import asyncio
import os
from pathlib import Path
from sqlalchemy import pool
from sqlalchemy.ext.asyncio import async_engine_from_config
from alembic import context
from app.repository import postgres_ssl_context
def migration_url() -> str:
path = os.getenv("BITRIX_SYNC_MIGRATION_DATABASE_URL_FILE")
if not path:
raise RuntimeError("BITRIX_SYNC_MIGRATION_DATABASE_URL_FILE is required")
value = Path(path).read_text(encoding="utf-8").rstrip("\r\n")
if not value:
raise RuntimeError("Bitrix migration database URL file is empty")
return value
def run_migrations_offline() -> None:
context.configure(
url=migration_url(),
target_metadata=None,
literal_binds=True,
dialect_opts={"paramstyle": "named"},
)
with context.begin_transaction():
context.run_migrations()
async def run_async_migrations() -> None:
configuration = context.config.get_section(context.config.config_ini_section) or {}
configuration["sqlalchemy.url"] = migration_url()
engine = async_engine_from_config(
configuration,
prefix="sqlalchemy.",
poolclass=pool.NullPool,
connect_args={"ssl": postgres_ssl_context()},
)
def run_sync_migrations(connection) -> None:
context.configure(connection=connection, target_metadata=None, compare_type=True)
with context.begin_transaction():
context.run_migrations()
async with engine.connect() as connection:
await connection.run_sync(run_sync_migrations)
await engine.dispose()
if context.is_offline_mode():
run_migrations_offline()
else:
asyncio.run(run_async_migrations())
@@ -0,0 +1,20 @@
"""Preserve the legacy connectivity-stub Alembic revision.
Revision ID: 0001_sync_baseline
Revises:
"""
from collections.abc import Sequence
revision: str = "0001_sync_baseline"
down_revision: str | None = None
branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None
def upgrade() -> None:
"""The legacy connectivity stub owned no runtime tables."""
def downgrade() -> None:
"""The legacy connectivity stub owned no runtime tables."""
@@ -0,0 +1,356 @@
"""Create durable bitrix_sync schema and contracts.
Revision ID: 0001_bitrix_sync_full
Revises: 0001_sync_baseline
Create Date: 2026-08-06
"""
from collections.abc import Sequence
from alembic import op
revision: str = "0001_bitrix_sync_full"
down_revision: str | None = "0001_sync_baseline"
branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None
def _execute_script(script: str) -> None:
"""Execute simple DDL statements separately for asyncpg compatibility."""
for statement in script.split(";"):
if statement.strip():
op.execute(statement)
def upgrade() -> None:
op.execute("CREATE EXTENSION IF NOT EXISTS pgcrypto")
op.execute("CREATE SCHEMA IF NOT EXISTS bitrix_sync")
_execute_script(
"""
CREATE TABLE bitrix_sync.workflow_instances (
id uuid PRIMARY KEY,
workflow_type varchar(64) NOT NULL CHECK (workflow_type IN
('contact.map_or_create','contact.update','contact.deactivate','contact.rebind',
'contact.webhook','contact.reconciliation','alert.reconciliation')),
user_id uuid,
external_id varchar(128),
state varchar(24) NOT NULL CHECK (state IN
('created','running','waiting_crm','waiting_retry','waiting_manual',
'succeeded','failed','cancelled')),
current_step varchar(64) NOT NULL,
source_task_id uuid UNIQUE,
deadline_at timestamptz NOT NULL,
outcome varchar(64),
completed_at timestamptz,
created_at timestamptz NOT NULL DEFAULT now(),
updated_at timestamptz NOT NULL DEFAULT now()
);
CREATE INDEX ix_workflow_claim
ON bitrix_sync.workflow_instances(state,updated_at)
WHERE state IN ('created','running','waiting_crm','waiting_retry');
CREATE TABLE bitrix_sync.entity_external_mapping (
id uuid PRIMARY KEY,
entity_type varchar(64) NOT NULL,
entity_id uuid NOT NULL,
external_system varchar(32) NOT NULL DEFAULT 'bitrix24'
CHECK (external_system='bitrix24'),
external_entity_type varchar(32) NOT NULL DEFAULT 'contact'
CHECK (external_entity_type='contact'),
external_id varchar(128) NOT NULL,
status varchar(16) NOT NULL CHECK (status IN ('active','closed','broken')),
opened_at timestamptz NOT NULL,
closed_at timestamptz,
close_reason varchar(64),
workflow_id uuid REFERENCES bitrix_sync.workflow_instances(id),
created_at timestamptz NOT NULL DEFAULT now(),
updated_at timestamptz NOT NULL DEFAULT now(),
CHECK ((status='active' AND closed_at IS NULL) OR
(status IN ('closed','broken')
AND (closed_at IS NOT NULL OR close_reason IS NOT NULL)))
);
CREATE UNIQUE INDEX uq_mapping_active_entity
ON bitrix_sync.entity_external_mapping(external_system,entity_type,entity_id)
WHERE status='active';
CREATE UNIQUE INDEX uq_mapping_active_external
ON bitrix_sync.entity_external_mapping
(external_system,external_entity_type,external_id)
WHERE status='active';
CREATE INDEX ix_mapping_history
ON bitrix_sync.entity_external_mapping(entity_id,opened_at DESC);
CREATE TABLE bitrix_sync.crm_commands (
id uuid PRIMARY KEY,
workflow_id uuid NOT NULL REFERENCES bitrix_sync.workflow_instances(id),
command_type varchar(40) NOT NULL CHECK (command_type IN
('duplicate_find','contact_get','contact_add','contact_update',
'citizenship_fields_get','contact_incremental_list',
'alert_get','alert_add','alert_update',
'rebind_target_get','rebind_old_get')),
safe_request jsonb NOT NULL DEFAULT '{}',
status varchar(24) NOT NULL CHECK (status IN
('pending','leased','in_flight','succeeded','retry','retry_wait',
'uncertain','reconcile','dead_letter','permanent','rate_limited')),
attempt_count integer NOT NULL DEFAULT 0 CHECK (attempt_count>=0),
next_attempt_at timestamptz NOT NULL DEFAULT now(),
locked_by varchar(128),
locked_until timestamptz,
lease_token uuid,
batch_id uuid,
correlation_id uuid,
safe_response jsonb,
safe_error_code varchar(64),
http_status integer,
completed_at timestamptz,
created_at timestamptz NOT NULL DEFAULT now(),
updated_at timestamptz NOT NULL DEFAULT now()
);
CREATE INDEX ix_crm_command_claim
ON bitrix_sync.crm_commands(status,next_attempt_at,created_at);
CREATE INDEX ix_crm_command_workflow
ON bitrix_sync.crm_commands(workflow_id,created_at);
CREATE TABLE bitrix_sync.webhook_inbox (
id uuid PRIMARY KEY,
receiver_type varchar(16) NOT NULL CHECK (receiver_type IN ('contact','alert')),
event_type varchar(64) NOT NULL,
event_id varchar(255),
source_timestamp timestamptz,
external_entity_id varchar(128) NOT NULL,
dedup_fingerprint varchar(64),
source_ip inet,
status varchar(16) NOT NULL CHECK (status IN
('received','coalesced','processing','processed','retry_wait','dead_letter')),
coalesced_count integer NOT NULL DEFAULT 1,
attempt_count integer NOT NULL DEFAULT 0,
next_attempt_at timestamptz NOT NULL DEFAULT now(),
locked_by varchar(128),
locked_until timestamptz,
lease_token uuid,
received_at timestamptz NOT NULL,
last_received_at timestamptz NOT NULL,
processed_at timestamptz,
safe_error_code varchar(64)
);
CREATE UNIQUE INDEX uq_webhook_event_id
ON bitrix_sync.webhook_inbox(receiver_type,event_id) WHERE event_id IS NOT NULL;
CREATE INDEX ix_webhook_claim
ON bitrix_sync.webhook_inbox(status,next_attempt_at,received_at);
CREATE UNIQUE INDEX uq_webhook_active_entity
ON bitrix_sync.webhook_inbox(receiver_type,external_entity_id)
WHERE status IN ('received','processing','retry_wait');
"""
)
_execute_script(
"""
CREATE SEQUENCE bitrix_sync.business_alert_number_seq;
CREATE TABLE bitrix_sync.business_alerts (
id uuid PRIMARY KEY,
alert_number bigint NOT NULL DEFAULT nextval('bitrix_sync.business_alert_number_seq'),
fingerprint varchar(64) NOT NULL,
alert_type varchar(64) NOT NULL,
severity varchar(16) NOT NULL CHECK (severity IN ('info','warning','critical')),
app_user_id uuid,
current_external_id varchar(128),
selected_external_id varchar(128),
candidate_external_ids text[] NOT NULL DEFAULT '{}',
remote_item_id varchar(128),
remote_stage_id varchar(128),
previous_alert_id uuid REFERENCES bitrix_sync.business_alerts(id),
workflow_id uuid REFERENCES bitrix_sync.workflow_instances(id),
status varchar(24) NOT NULL CHECK (status IN
('open','in_progress','resolved','closed_without_resolution','remote_missing')),
occurrence_count integer NOT NULL DEFAULT 1,
first_occurred_at timestamptz NOT NULL,
last_occurred_at timestamptz NOT NULL,
created_at timestamptz NOT NULL DEFAULT now(),
updated_at timestamptz NOT NULL DEFAULT now()
);
CREATE UNIQUE INDEX uq_business_alert_open
ON bitrix_sync.business_alerts(alert_type,fingerprint) WHERE status='open';
CREATE INDEX ix_business_alert_remote
ON bitrix_sync.business_alerts(remote_item_id) WHERE remote_item_id IS NOT NULL;
CREATE TABLE bitrix_sync.rebind_requests (
id uuid PRIMARY KEY,
user_id uuid NOT NULL,
old_external_id varchar(128),
target_external_id varchar(128) NOT NULL,
reason varchar(500) NOT NULL,
operator_id varchar(128) NOT NULL,
workflow_id uuid NOT NULL UNIQUE REFERENCES bitrix_sync.workflow_instances(id),
status varchar(24) NOT NULL CHECK (status IN
('pending','processing','retry_wait','succeeded','failed','cancelled')),
safe_error_code varchar(64),
requested_at timestamptz NOT NULL DEFAULT now(),
completed_at timestamptz,
created_at timestamptz NOT NULL DEFAULT now(),
updated_at timestamptz NOT NULL DEFAULT now()
);
CREATE INDEX ix_rebind_claim
ON bitrix_sync.rebind_requests(status,requested_at)
WHERE status IN ('pending','retry_wait');
CREATE TABLE bitrix_sync.contact_snapshots (
id uuid PRIMARY KEY,
mapping_id uuid NOT NULL REFERENCES bitrix_sync.entity_external_mapping(id),
user_id uuid NOT NULL,
external_id varchar(128) NOT NULL,
full_name_hash varchar(64),
email_hash varchar(64),
phone_hash varchar(64),
citizenship_hash varchar(64),
citizenship_enum_id varchar(128),
citizenship_dictionary_loaded_at timestamptz,
source_updated_at timestamptz,
app_version varchar(128),
last_applied_source varchar(24) NOT NULL CHECK (last_applied_source IN
('webhook','reconciliation','app_create','app_update')),
last_webhook_received_at timestamptz,
last_webhook_source_at timestamptz,
profile_stale boolean NOT NULL DEFAULT false,
created_at timestamptz NOT NULL DEFAULT now(),
updated_at timestamptz NOT NULL DEFAULT now(),
UNIQUE(mapping_id)
);
CREATE INDEX ix_contact_snapshot_source
ON bitrix_sync.contact_snapshots(source_updated_at);
CREATE TABLE bitrix_sync.settings_versions (
id uuid PRIMARY KEY,
version bigint NOT NULL UNIQUE,
validation_status varchar(16) NOT NULL
CHECK (validation_status IN ('pending','valid','invalid')),
validation_errors jsonb NOT NULL DEFAULT '[]',
active boolean NOT NULL DEFAULT false,
created_by varchar(128) NOT NULL,
created_at timestamptz NOT NULL DEFAULT now(),
activated_at timestamptz
);
CREATE UNIQUE INDEX uq_settings_version_active
ON bitrix_sync.settings_versions(active) WHERE active=true;
CREATE TABLE bitrix_sync.settings (
id uuid PRIMARY KEY,
version_id uuid NOT NULL REFERENCES bitrix_sync.settings_versions(id),
key varchar(128) NOT NULL,
value_type varchar(16) NOT NULL
CHECK (value_type IN ('integer','number','boolean','object')),
value_json jsonb NOT NULL,
validation_status varchar(16) NOT NULL CHECK (validation_status IN ('valid','invalid')),
active boolean NOT NULL DEFAULT false,
created_at timestamptz NOT NULL DEFAULT now(),
updated_at timestamptz NOT NULL DEFAULT now(),
UNIQUE(version_id,key)
);
CREATE INDEX ix_settings_active ON bitrix_sync.settings(key) WHERE active=true;
CREATE TABLE bitrix_sync.technical_dead_letters (
id uuid PRIMARY KEY,
workflow_id uuid REFERENCES bitrix_sync.workflow_instances(id),
command_id uuid REFERENCES bitrix_sync.crm_commands(id),
operation varchar(64) NOT NULL,
safe_error_code varchar(64) NOT NULL,
attempt_count integer NOT NULL DEFAULT 0,
deadline_at timestamptz,
correlation_id uuid,
failed_at timestamptz NOT NULL,
created_at timestamptz NOT NULL DEFAULT now()
);
CREATE INDEX ix_technical_dlq_failed
ON bitrix_sync.technical_dead_letters(failed_at DESC);
CREATE TABLE bitrix_sync.reconciliation_cursors (
job_type varchar(64) PRIMARY KEY,
watermark timestamptz NOT NULL,
overlap_seconds integer NOT NULL CHECK (overlap_seconds>=0),
last_success_at timestamptz,
last_scanned_count integer NOT NULL DEFAULT 0,
last_updated_count integer NOT NULL DEFAULT 0,
recovered_without_webhook_count integer NOT NULL DEFAULT 0,
created_at timestamptz NOT NULL DEFAULT now(),
updated_at timestamptz NOT NULL DEFAULT now()
);
CREATE TABLE bitrix_sync.limiter_coordination (
limiter_key varchar(128) PRIMARY KEY,
tokens numeric(12,6) NOT NULL,
capacity numeric(12,6) NOT NULL CHECK (capacity>0),
refill_per_second numeric(12,6) NOT NULL CHECK (refill_per_second>0),
updated_at timestamptz NOT NULL,
blocked_until timestamptz,
method_class_blocks jsonb NOT NULL DEFAULT '{}',
fencing_token bigint NOT NULL DEFAULT 0
);
CREATE TABLE bitrix_sync.citizenship_dictionary (
enum_id varchar(128) PRIMARY KEY,
display_value varchar(255) NOT NULL,
loaded_at timestamptz NOT NULL,
expires_at timestamptz NOT NULL
);
"""
)
op.execute(
"""
CREATE OR REPLACE FUNCTION bitrix_sync.request_bitrix_contact_rebind(
p_user_id uuid,
p_target_b24_id varchar,
p_reason varchar,
p_operator_id varchar
) RETURNS uuid
LANGUAGE plpgsql
SECURITY DEFINER
SET search_path=bitrix_sync,pg_temp
AS $$
DECLARE
v_old_id varchar(128);
v_request_id uuid := gen_random_uuid();
v_workflow_id uuid := gen_random_uuid();
BEGIN
IF p_target_b24_id !~ '^[1-9][0-9]*$'
OR length(trim(p_reason)) < 5
OR length(trim(p_operator_id)) < 1 THEN
RAISE EXCEPTION 'invalid rebind request' USING ERRCODE='22023';
END IF;
SELECT external_id INTO v_old_id
FROM bitrix_sync.entity_external_mapping
WHERE entity_id=p_user_id AND entity_type='contact'
AND external_system='bitrix24' AND status='active'
FOR UPDATE;
IF EXISTS (
SELECT 1 FROM bitrix_sync.entity_external_mapping
WHERE external_system='bitrix24' AND external_entity_type='contact'
AND external_id=p_target_b24_id AND status='active'
AND entity_id<>p_user_id
) THEN
RAISE EXCEPTION 'target contact has another active mapping'
USING ERRCODE='23505';
END IF;
INSERT INTO bitrix_sync.workflow_instances
(id,workflow_type,user_id,external_id,state,current_step,deadline_at,created_at,updated_at)
VALUES
(v_workflow_id,'contact.rebind',p_user_id,p_target_b24_id,'created',
'validate_contacts',now()+interval '24 hours',now(),now());
INSERT INTO bitrix_sync.rebind_requests
(id,user_id,old_external_id,target_external_id,reason,operator_id,
workflow_id,status,requested_at,created_at,updated_at)
VALUES
(v_request_id,p_user_id,v_old_id,p_target_b24_id,trim(p_reason),
trim(p_operator_id),v_workflow_id,'pending',now(),now(),now());
RETURN v_request_id;
END;
$$;
"""
)
op.execute(
"""
REVOKE ALL ON FUNCTION
bitrix_sync.request_bitrix_contact_rebind(uuid,varchar,varchar,varchar)
FROM PUBLIC
"""
)
def downgrade() -> None:
raise RuntimeError("bitrix_sync production migration is forward-only")
@@ -0,0 +1,85 @@
"""Adopt the App queue contract and migrate legacy mappings.
Revision ID: 0002_app_queue_contract
Revises: 0001_bitrix_sync_full
Create Date: 2026-08-06
"""
from collections.abc import Sequence
from alembic import op
revision: str = "0002_app_queue_contract"
down_revision: str | None = "0001_bitrix_sync_full"
branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None
def upgrade() -> None:
op.execute(
"""
DO $$
BEGIN
IF to_regclass('han_app.entity_external_mapping') IS NOT NULL THEN
EXECUTE $copy$
INSERT INTO bitrix_sync.entity_external_mapping
(id,entity_type,entity_id,external_system,external_entity_type,
external_id,status,opened_at,closed_at,close_reason,created_at,updated_at)
SELECT id,entity_type,entity_id,'bitrix24','contact',
external_id,'active',coalesce(created_at,now()),NULL,NULL,
coalesce(created_at,now()),coalesce(created_at,now())
FROM han_app.entity_external_mapping
ON CONFLICT DO NOTHING
$copy$;
IF EXISTS (
SELECT 1 FROM han_app.entity_external_mapping old
LEFT JOIN bitrix_sync.entity_external_mapping new ON new.id=old.id
WHERE new.id IS NULL
) THEN
RAISE EXCEPTION 'legacy mapping migration verification failed';
END IF;
END IF;
END $$;
"""
)
op.execute(
"""
INSERT INTO bitrix_sync.settings_versions
(id,version,validation_status,active,created_by,created_at,activated_at)
VALUES ('00000000-0000-0000-0000-000000000001',1,'valid',true,'migration',now(),now())
ON CONFLICT DO NOTHING
"""
)
op.execute(
"""
INSERT INTO bitrix_sync.settings
(id,version_id,key,value_type,value_json,validation_status,active,created_at,updated_at)
VALUES
(gen_random_uuid(),'00000000-0000-0000-0000-000000000001','worker','object',
jsonb_build_object(
'batch_size',20,'batch_wait_ms',200,'claim_size',20,'lease_seconds',60,
'limiter_refill_per_sec',2,'limiter_burst',2,'max_in_flight',2,
'retry_base_seconds',1,'retry_max_seconds',900,'retry_horizon_seconds',86400
),
'valid',true,now(),now()),
(gen_random_uuid(),'00000000-0000-0000-0000-000000000001','reconciliation','object',
jsonb_build_object(
'contact_interval_seconds',900,'alert_interval_seconds',3600,
'overlap_seconds',300,'recovered_spike_threshold',20
),
'valid',true,now(),now()),
(gen_random_uuid(),'00000000-0000-0000-0000-000000000001','business_alerts','object',
jsonb_build_object(
'entity_type_id',NULL,'category_id',NULL,'stage_new',NULL,
'stage_in_progress',NULL,'stage_resolved',NULL,
'stage_closed_without_resolution',NULL,'sla_business_hours',8
),
'valid',true,now(),now())
ON CONFLICT DO NOTHING;
"""
)
def downgrade() -> None:
raise RuntimeError("App queue contract migration is forward-only")
@@ -0,0 +1 @@
"""HAN Bitrix24 synchronization service."""
@@ -0,0 +1,137 @@
from __future__ import annotations
import ipaddress
import re
from functools import cached_property
from pathlib import Path
from urllib.parse import urlsplit
from pydantic import Field, SecretStr, model_validator
from pydantic_settings import BaseSettings, SettingsConfigDict
FIELD_RE = re.compile(r"^UF_CRM_[0-9]+$")
MEMBER_RE = re.compile(r"^[A-Za-z0-9_-]{8,128}$")
def _read_secret(value: SecretStr | None, path: Path | None) -> SecretStr | None:
if value and value.get_secret_value():
return value
if path:
text = path.read_text(encoding="utf-8").strip()
return SecretStr(text) if text else None
return None
class Settings(BaseSettings):
model_config = SettingsConfigDict(env_prefix="BITRIX_SYNC_", extra="ignore")
enabled: bool = False
mode: str = "disabled"
database_url: SecretStr | None = None
database_url_file: Path | None = None
crm_rest_webhook_url: SecretStr | None = None
crm_rest_webhook_url_file: Path | None = None
contact_receiver_token: SecretStr | None = None
contact_receiver_token_file: Path | None = None
contact_receiver_previous_token: SecretStr | None = None
alert_receiver_token: SecretStr | None = None
alert_receiver_token_file: Path | None = None
alert_receiver_previous_token: SecretStr | None = None
service_token: SecretStr | None = None
service_token_file: Path | None = None
portal_host: str | None = None
portal_member_id: str | None = None
public_base_url: str | None = None
contact_user_id_field: str | None = None
contact_registered_field: str | None = None
contact_citizenship_field: str | None = None
webhook_allowed_cidrs: str = ""
http_timeout_sec: float = Field(default=10, ge=1, le=60)
db_pool_size: int = Field(default=5, ge=1, le=30)
webhook_max_body_bytes: int = Field(default=16_384, ge=1024, le=65_536)
webhook_max_fields: int = Field(default=24, ge=8, le=64)
lease_seconds: int = Field(default=60, ge=10, le=600)
claim_size: int = Field(default=20, ge=1, le=100)
batch_size: int = Field(default=20, ge=1, le=50)
batch_wait_ms: int = Field(default=200, ge=10, le=5000)
limiter_refill_per_sec: float = Field(default=2, gt=0, le=50)
limiter_burst: int = Field(default=2, ge=1, le=50)
max_in_flight: int = Field(default=2, ge=1, le=20)
retry_base_seconds: float = Field(default=1, ge=0.1, le=60)
retry_max_seconds: float = Field(default=900, ge=1, le=3600)
retry_horizon_seconds: int = Field(default=86_400, ge=60, le=604_800)
reconciliation_overlap_seconds: int = Field(default=300, ge=0, le=3600)
reconciliation_interval_seconds: int = Field(default=900, ge=60, le=86_400)
@model_validator(mode="after")
def validate_mode(self) -> Settings:
self.database_url = _read_secret(self.database_url, self.database_url_file)
self.crm_rest_webhook_url = _read_secret(
self.crm_rest_webhook_url, self.crm_rest_webhook_url_file
)
self.contact_receiver_token = _read_secret(
self.contact_receiver_token, self.contact_receiver_token_file
)
self.alert_receiver_token = _read_secret(
self.alert_receiver_token, self.alert_receiver_token_file
)
self.service_token = _read_secret(self.service_token, self.service_token_file)
expected_mode = "full" if self.enabled else "disabled"
if self.mode != expected_mode:
raise ValueError(f"mode must be {expected_mode!r} when enabled={self.enabled}")
if not self.enabled:
return self
required = {
"database_url": self.database_url,
"crm_rest_webhook_url": self.crm_rest_webhook_url,
"contact_receiver_token": self.contact_receiver_token,
"alert_receiver_token": self.alert_receiver_token,
"service_token": self.service_token,
"portal_host": self.portal_host,
"portal_member_id": self.portal_member_id,
"public_base_url": self.public_base_url,
"contact_user_id_field": self.contact_user_id_field,
"contact_registered_field": self.contact_registered_field,
"contact_citizenship_field": self.contact_citizenship_field,
}
missing = [name for name, value in required.items() if not value]
if missing:
raise ValueError("missing full-mode settings: " + ", ".join(missing))
for name in (
"contact_user_id_field",
"contact_registered_field",
"contact_citizenship_field",
):
if not FIELD_RE.fullmatch(str(getattr(self, name))):
raise ValueError(f"{name} must match UF_CRM_<digits>")
if not MEMBER_RE.fullmatch(str(self.portal_member_id)):
raise ValueError("portal_member_id has invalid format")
crm = urlsplit(self.crm_rest_webhook_url.get_secret_value())
public = urlsplit(str(self.public_base_url))
if crm.scheme != "https" or crm.hostname != self.portal_host or crm.port not in (None, 443):
raise ValueError("CRM URL must be HTTPS on the approved portal host")
if public.scheme != "https" or not public.hostname or public.query or public.fragment:
raise ValueError("public_base_url must be a query-free HTTPS origin")
if not self.allowed_networks:
raise ValueError("webhook_allowed_cidrs cannot be empty in full mode")
return self
@cached_property
def allowed_networks(self) -> tuple[ipaddress.IPv4Network | ipaddress.IPv6Network, ...]:
values = [item.strip() for item in self.webhook_allowed_cidrs.split(",") if item.strip()]
return tuple(ipaddress.ip_network(item, strict=True) for item in values)
@staticmethod
def rest_field_name(field: str) -> str:
if not FIELD_RE.fullmatch(field):
raise ValueError("invalid Bitrix custom field")
return "ufCrm_" + field.removeprefix("UF_CRM_")
def load_settings() -> Settings:
return Settings()
@@ -0,0 +1,185 @@
from __future__ import annotations
import asyncio
import ssl
from dataclasses import dataclass
from datetime import UTC, datetime
from enum import StrEnum
from typing import Any
from urllib.parse import urljoin, urlsplit
import httpx
class CrmOutcome(StrEnum):
SUCCEEDED = "succeeded"
RETRY = "retry"
UNCERTAIN = "uncertain"
PERMANENT = "permanent"
RATE_LIMITED = "rate_limited"
@dataclass(frozen=True)
class CrmResult:
outcome: CrmOutcome
result: Any = None
error_code: str | None = None
http_status: int | None = None
retry_after: float | None = None
class CrmClient:
"""Host-locked, verified-TLS Bitrix client; redirects are never followed."""
def __init__(self, credential_url: str, approved_host: str, timeout: float) -> None:
parsed = urlsplit(credential_url)
if parsed.scheme != "https" or parsed.hostname != approved_host:
raise ValueError("credential URL is outside approved Bitrix host")
self._base_url = credential_url.rstrip("/") + "/"
self._host = approved_host
self._client = httpx.AsyncClient(
timeout=httpx.Timeout(timeout),
verify=ssl.create_default_context(),
follow_redirects=False,
limits=httpx.Limits(max_connections=4, max_keepalive_connections=2),
headers={"Accept": "application/json"},
)
async def close(self) -> None:
await self._client.aclose()
async def call(self, method: str, params: dict[str, Any], *, mutating: bool) -> CrmResult:
if method not in ALLOWED_METHODS:
raise ValueError("unapproved CRM method")
url = urljoin(self._base_url, method + ".json")
if urlsplit(url).hostname != self._host:
raise ValueError("CRM host changed during URL construction")
try:
response = await self._client.post(url, json=params)
except (httpx.ConnectError, httpx.ReadError, httpx.RemoteProtocolError):
return CrmResult(CrmOutcome.RETRY, error_code="crm_network")
except httpx.TimeoutException:
outcome = CrmOutcome.UNCERTAIN if mutating else CrmOutcome.RETRY
return CrmResult(outcome, error_code="crm_timeout")
if response.is_redirect:
return CrmResult(
CrmOutcome.PERMANENT,
error_code="crm_redirect_rejected",
http_status=response.status_code,
)
retry_after = _retry_after(response)
if response.status_code in (408, 429) or response.status_code >= 500:
return CrmResult(
CrmOutcome.RETRY,
error_code=f"crm_http_{response.status_code}",
http_status=response.status_code,
retry_after=retry_after,
)
try:
payload = response.json()
except ValueError:
return CrmResult(
CrmOutcome.PERMANENT,
error_code="crm_malformed_response",
http_status=response.status_code,
)
error = payload.get("error")
if error == "QUERY_LIMIT_EXCEEDED":
return CrmResult(CrmOutcome.RATE_LIMITED, error_code=error, retry_after=retry_after)
if error == "OPERATION_TIME_LIMIT":
return CrmResult(CrmOutcome.RETRY, error_code=error, retry_after=retry_after)
if error:
permanent = error in {
"ERROR_METHOD_NOT_FOUND",
"ERROR_WRONG_AUTH_TYPE",
"INVALID_CREDENTIALS",
"ACCESS_DENIED",
"ERROR_ARGUMENT",
}
return CrmResult(
CrmOutcome.PERMANENT if permanent else CrmOutcome.RETRY,
error_code=str(error)[:64],
http_status=response.status_code,
)
return CrmResult(CrmOutcome.SUCCEEDED, result=payload.get("result"))
async def batch(self, commands: list[tuple[str, dict[str, Any]]]) -> list[CrmResult]:
if not commands:
return []
if len(commands) > 50:
raise ValueError("Bitrix batch limit exceeded")
cmd = {
str(index): f"{method}?{httpx.QueryParams(params)}"
for index, (method, params) in enumerate(commands)
if method in ALLOWED_METHODS
}
batch = await self.call("batch", {"halt": 0, "cmd": cmd}, mutating=True)
if batch.outcome != CrmOutcome.SUCCEEDED:
return [batch for _ in commands]
result = batch.result or {}
successes = result.get("result", {})
errors = result.get("result_error", {})
return [
CrmResult(CrmOutcome.SUCCEEDED, result=successes.get(str(i)))
if str(i) in successes
else CrmResult(CrmOutcome.RETRY, error_code=str(errors.get(str(i), "batch_missing")))
for i in range(len(commands))
]
ALLOWED_METHODS = frozenset(
{
"batch",
"crm.duplicate.findbycomm",
"crm.contact.get",
"crm.contact.add",
"crm.contact.update",
"crm.contact.userfield.list",
"crm.item.list",
"crm.item.get",
"crm.item.add",
"crm.item.update",
}
)
def _retry_after(response: httpx.Response) -> float | None:
value = response.headers.get("Retry-After")
if not value:
return None
try:
return max(0.0, float(value))
except ValueError:
try:
retry_at = datetime.fromisoformat(value).astimezone(UTC)
return max(0.0, (retry_at - datetime.now(UTC)).total_seconds())
except ValueError:
return None
class TokenBucket:
def __init__(self, refill_per_second: float, burst: int, max_in_flight: int) -> None:
self.refill_per_second = refill_per_second
self.burst = float(burst)
self.tokens = float(burst)
self.updated_at = asyncio.get_running_loop().time()
self._lock = asyncio.Lock()
self._slots = asyncio.Semaphore(max_in_flight)
async def acquire(self) -> None:
await self._slots.acquire()
while True:
async with self._lock:
now = asyncio.get_running_loop().time()
self.tokens = min(
self.burst, self.tokens + (now - self.updated_at) * self.refill_per_second
)
self.updated_at = now
if self.tokens >= 1:
self.tokens -= 1
return
delay = (1 - self.tokens) / self.refill_per_second
await asyncio.sleep(delay)
def release(self) -> None:
self._slots.release()
@@ -0,0 +1,106 @@
from __future__ import annotations
import hashlib
import random
import re
from dataclasses import dataclass
from datetime import UTC, datetime
from enum import StrEnum
from typing import Any
PHONE_RE = re.compile(r"^\+7[0-9]{10}$")
EMAIL_RE = re.compile(r"^[^@\s]+@[^@\s]+\.[^@\s]+$")
class WorkflowState(StrEnum):
CREATED = "created"
RUNNING = "running"
WAITING_CRM = "waiting_crm"
WAITING_RETRY = "waiting_retry"
WAITING_MANUAL = "waiting_manual"
SUCCEEDED = "succeeded"
FAILED = "failed"
CANCELLED = "cancelled"
TRANSITIONS: dict[WorkflowState, frozenset[WorkflowState]] = {
WorkflowState.CREATED: frozenset({WorkflowState.RUNNING, WorkflowState.CANCELLED}),
WorkflowState.RUNNING: frozenset(
{
WorkflowState.WAITING_CRM,
WorkflowState.WAITING_RETRY,
WorkflowState.WAITING_MANUAL,
WorkflowState.SUCCEEDED,
WorkflowState.FAILED,
WorkflowState.CANCELLED,
}
),
WorkflowState.WAITING_CRM: frozenset(
{
WorkflowState.RUNNING,
WorkflowState.WAITING_RETRY,
WorkflowState.WAITING_MANUAL,
WorkflowState.FAILED,
}
),
WorkflowState.WAITING_RETRY: frozenset(
{WorkflowState.RUNNING, WorkflowState.WAITING_MANUAL, WorkflowState.FAILED}
),
WorkflowState.WAITING_MANUAL: frozenset(
{WorkflowState.RUNNING, WorkflowState.CANCELLED}
),
WorkflowState.SUCCEEDED: frozenset(),
WorkflowState.FAILED: frozenset(),
WorkflowState.CANCELLED: frozenset(),
}
def assert_transition(current: WorkflowState, target: WorkflowState) -> None:
if target not in TRANSITIONS[current]:
raise ValueError(f"forbidden workflow transition {current} -> {target}")
@dataclass(frozen=True)
class ContactCandidate:
b24_id: str
created_at: datetime
crm_user_id: str | None
def validate_phone(value: str) -> str:
if not PHONE_RE.fullmatch(value):
raise ValueError("phone must be Russian E.164 +7XXXXXXXXXX")
return value
def choose_newest(candidates: list[ContactCandidate]) -> ContactCandidate | None:
if not candidates:
return None
return max(candidates, key=lambda item: (item.created_at, int(item.b24_id)))
def select_email(items: list[dict[str, Any]]) -> str | None:
valid = [
item
for item in items
if isinstance(item.get("VALUE"), str) and EMAIL_RE.fullmatch(item["VALUE"])
]
work = next((item for item in valid if item.get("VALUE_TYPE") == "WORK"), None)
selected = work or (valid[0] if valid else None)
return selected["VALUE"] if selected else None
def safe_hash(value: str | None) -> str | None:
return hashlib.sha256(value.encode()).hexdigest() if value is not None else None
def full_jitter_delay(
attempt: int, base: float, maximum: float, *, rng: random.Random | None = None
) -> float:
ceiling = min(maximum, base * (2 ** max(0, attempt - 1)))
return (rng or random.SystemRandom()).uniform(0, ceiling)
def parse_crm_datetime(value: str) -> datetime:
parsed = datetime.fromisoformat(value.replace("Z", "+00:00"))
return parsed.astimezone(UTC)
@@ -0,0 +1,629 @@
from __future__ import annotations
import asyncio
import uuid
from dataclasses import dataclass, field
from typing import Any
from sqlalchemy import text
from app.config import Settings
from app.crm import CrmClient, CrmOutcome, CrmResult
from app.domain import (
ContactCandidate,
choose_newest,
full_jitter_delay,
parse_crm_datetime,
select_email,
validate_phone,
)
from app.repository import LeasedTask, LeasedWebhook, Profile, Repository
class BusinessConflict(Exception):
def __init__(self, code: str, candidates: list[str] | None = None) -> None:
self.code = code
self.candidates = candidates or []
super().__init__(code)
@dataclass
class WorkflowEngine:
repository: Repository
crm: CrmClient
settings: Settings
_in_flight: asyncio.Semaphore = field(init=False, repr=False)
def __post_init__(self) -> None:
self._in_flight = asyncio.Semaphore(self.settings.max_in_flight)
async def process(self, task: LeasedTask) -> None:
workflow_id = await self.repository.create_workflow(task)
profile = await self.repository.load_profile(task.user_id)
if profile is None:
await self._manual(workflow_id, "profile_missing", task.user_id)
await self.repository.complete_task(task, workflow_id)
return
try:
if task.task_type == "contact.map_or_create":
await self._map_or_create(workflow_id, profile)
elif task.task_type == "contact.update":
await self._update(workflow_id, profile)
elif task.task_type == "contact.deactivate":
await self._deactivate(workflow_id, profile)
else:
await self._technical_failure(workflow_id, "unknown_task_type")
await self.repository.complete_task(task, workflow_id)
except BusinessConflict as exc:
await self._alert(workflow_id, task.user_id, exc.code, exc.candidates)
await self._manual(workflow_id, exc.code, task.user_id)
await self.repository.complete_task(task, workflow_id)
except RetryableWorkflow as exc:
delay = exc.retry_after or full_jitter_delay(
task.attempt_count + 1,
self.settings.retry_base_seconds,
self.settings.retry_max_seconds,
)
await self.repository.retry_task(task, exc.code, delay)
async def process_webhook(self, item: LeasedWebhook) -> None:
if item.receiver_type == "alert":
await self.repository.complete_webhook(item)
return
user_id = await self.repository.mapped_user_for_external(item.external_id)
if user_id is None:
await self.repository.complete_webhook(item)
return
workflow_id = uuid.uuid4()
async with self.repository.transaction() as connection:
await connection.execute(
text(
"""
INSERT INTO bitrix_sync.workflow_instances
(id,workflow_type,user_id,external_id,state,current_step,deadline_at,
created_at,updated_at)
VALUES (:id,'contact.webhook',:user_id,:external_id,'running',
'read_contact',now()+interval '24 hours',now(),now())
"""
),
{"id": workflow_id, "user_id": user_id, "external_id": item.external_id},
)
try:
result = await self._command(
workflow_id,
"contact_get",
"crm.contact.get",
{"id": item.external_id, "select": self._select_fields()},
mutating=False,
)
contact = result.result
if str(contact.get(self.settings.contact_user_id_field)) != str(user_id):
await self._alert(
workflow_id,
user_id,
"mapping_identity_mismatch",
[item.external_id],
)
await self._manual(workflow_id, "mapping_identity_mismatch", user_id)
await self.repository.complete_webhook(item)
return
citizenship = await self._resolve_citizenship(
workflow_id, contact.get(self.settings.contact_citizenship_field)
)
source_value = contact.get("DATE_MODIFY") or contact.get("updatedTime")
await self.repository.apply_crm_profile(
user_id,
item.external_id,
full_name=contact.get("NAME") or None,
citizenship=citizenship,
email=select_email(contact.get("EMAIL") or []),
source_updated_at=parse_crm_datetime(source_value) if source_value else None,
source="reconciliation"
if item.event_type == "contact.reconciliation"
else "webhook",
)
await self.repository.complete_webhook(item)
async with self.repository.transaction() as connection:
await connection.execute(
text(
"""
UPDATE bitrix_sync.workflow_instances
SET state='succeeded',current_step='done',
completed_at=now(),updated_at=now()
WHERE id=:id
"""
),
{"id": workflow_id},
)
except RetryableWorkflow:
raise
async def _map_or_create(self, workflow_id: uuid.UUID, profile: Profile) -> None:
if profile.identity_status != "A" or profile.profile_status != "A":
await self._deactivate(workflow_id, profile)
return
validate_phone(profile.phone)
if mapped := await self.repository.active_mapping(profile.user_id):
await self._ensure_registered(workflow_id, mapped, profile.user_id)
return
found = await self._command(
workflow_id,
"duplicate_find",
"crm.duplicate.findbycomm",
{"type": "PHONE", "values": [profile.phone], "entity_type": "CONTACT"},
mutating=False,
)
ids = _contact_ids(found.result)
contacts: list[dict[str, Any]] = []
for contact_id in ids:
result = await self._command(
workflow_id,
"contact_get",
"crm.contact.get",
{"id": contact_id, "select": self._select_fields()},
mutating=False,
)
if isinstance(result.result, dict):
contacts.append(result.result)
same_user = next(
(
item
for item in contacts
if str(item.get(self.settings.contact_user_id_field)) == str(profile.user_id)
),
None,
)
alert_code: str | None = None
if same_user:
selected_id = str(same_user["ID"])
elif not contacts:
selected_id = await self._create_contact(workflow_id, profile)
else:
candidates = [
ContactCandidate(
str(item["ID"]),
parse_crm_datetime(item.get("CREATED_TIME", "1970-01-01T00:00:00Z")),
item.get(self.settings.contact_user_id_field),
)
for item in contacts
]
selected = choose_newest(candidates)
assert selected is not None
all_ids = [item.b24_id for item in candidates]
if selected.crm_user_id and selected.crm_user_id != str(profile.user_id):
selected_id = await self._create_contact(workflow_id, profile)
alert_code = "contact_owned_by_other_user"
else:
selected_id = selected.b24_id
await self._write_identity(workflow_id, selected_id, profile.user_id, active=True)
if len(candidates) > 1:
alert_code = "duplicate_contacts"
if alert_code:
await self._alert(workflow_id, profile.user_id, alert_code, all_ids)
await self._activate_mapping(workflow_id, profile.user_id, selected_id)
async def _create_contact(self, workflow_id: uuid.UUID, profile: Profile) -> str:
result = await self._command(
workflow_id,
"contact_add",
"crm.contact.add",
{
"fields": {
"PHONE": [{"VALUE": profile.phone, "VALUE_TYPE": "WORK"}],
self.settings.contact_user_id_field: str(profile.user_id),
self.settings.contact_registered_field: "1",
}
},
mutating=True,
)
return str(result.result)
async def _update(self, workflow_id: uuid.UUID, profile: Profile) -> None:
validate_phone(profile.phone)
mapping = await self.repository.active_mapping(profile.user_id)
if mapping is None:
await self._coalesce_map_or_create(profile.user_id)
return
await self._command(
workflow_id,
"contact_update",
"crm.contact.update",
{
"id": mapping,
"fields": {"PHONE": [{"VALUE": profile.phone, "VALUE_TYPE": "WORK"}]},
},
mutating=True,
)
async def _deactivate(self, workflow_id: uuid.UUID, profile: Profile) -> None:
mapping = await self.repository.active_mapping(profile.user_id)
if mapping is None:
return
await self._write_identity(workflow_id, mapping, profile.user_id, active=False)
async with self.repository.transaction() as connection:
await connection.execute(
text(
"""
UPDATE bitrix_sync.entity_external_mapping
SET status='closed', closed_at=now(), close_reason='deactivated',
workflow_id=:workflow_id, updated_at=now()
WHERE entity_id=:user_id AND external_id=:external_id AND status='active'
"""
),
{
"workflow_id": workflow_id,
"user_id": profile.user_id,
"external_id": mapping,
},
)
async def process_rebind(self, request_id: uuid.UUID) -> None:
async with self.repository.transaction() as connection:
request = (
await connection.execute(
text(
"""
SELECT id,user_id,old_external_id,target_external_id,workflow_id
FROM bitrix_sync.rebind_requests
WHERE id=:id AND status IN ('pending','retry_wait')
FOR UPDATE
"""
),
{"id": request_id},
)
).mappings().first()
if not request:
return
await connection.execute(
text(
"""
UPDATE bitrix_sync.rebind_requests
SET status='processing',updated_at=now() WHERE id=:id
"""
),
{"id": request_id},
)
target = await self._command(
request["workflow_id"],
"rebind_target_get",
"crm.contact.get",
{"id": request["target_external_id"], "select": self._select_fields()},
mutating=False,
)
target_user = target.result.get(self.settings.contact_user_id_field)
if target_user and target_user != str(request["user_id"]):
raise BusinessConflict("rebind_target_owned", [request["target_external_id"]])
await self._write_identity(
request["workflow_id"], request["target_external_id"], request["user_id"], active=True
)
if request["old_external_id"]:
old = await self._command(
request["workflow_id"],
"rebind_old_get",
"crm.contact.get",
{"id": request["old_external_id"], "select": self._select_fields()},
mutating=False,
)
if old.result.get(self.settings.contact_user_id_field) == str(request["user_id"]):
await self._write_identity(
request["workflow_id"],
request["old_external_id"],
request["user_id"],
active=False,
)
async with self.repository.transaction() as connection:
await connection.execute(
text(
"""
UPDATE bitrix_sync.entity_external_mapping
SET status='closed',closed_at=now(),close_reason='rebind',
workflow_id=:workflow_id,updated_at=now()
WHERE entity_id=:user_id AND status='active'
"""
),
{
"workflow_id": request["workflow_id"],
"user_id": request["user_id"],
},
)
await connection.execute(
text(
"""
INSERT INTO bitrix_sync.entity_external_mapping
(id,entity_type,entity_id,external_system,external_entity_type,
external_id,status,opened_at,workflow_id,created_at,updated_at)
VALUES (gen_random_uuid(),'contact',:user_id,'bitrix24','contact',
:target,'active',now(),:workflow_id,now(),now())
"""
),
{
"workflow_id": request["workflow_id"],
"user_id": request["user_id"],
"target": request["target_external_id"],
},
)
await connection.execute(
text(
"""
UPDATE bitrix_sync.rebind_requests
SET status='succeeded',completed_at=now(),updated_at=now()
WHERE id=:request_id
"""
),
{
"request_id": request_id,
},
)
async def _ensure_registered(
self, workflow_id: uuid.UUID, external_id: str, user_id: uuid.UUID
) -> None:
contact = await self._command(
workflow_id,
"contact_get",
"crm.contact.get",
{"id": external_id, "select": self._select_fields()},
mutating=False,
)
if str(contact.result.get(self.settings.contact_registered_field)) not in {"1", "Y"}:
await self._write_identity(workflow_id, external_id, user_id, active=True)
async def _resolve_citizenship(
self, workflow_id: uuid.UUID, enum_id: str | int | None
) -> str | None:
if enum_id in (None, ""):
return None
async with self.repository.transaction() as connection:
value = (
await connection.execute(
text(
"""
SELECT display_value FROM bitrix_sync.citizenship_dictionary
WHERE enum_id=:enum_id AND expires_at>now()
"""
),
{"enum_id": str(enum_id)},
)
).scalar_one_or_none()
if value is not None:
return value
result = await self._command(
workflow_id,
"citizenship_fields_get",
"crm.contact.userfield.list",
{"filter": {"FIELD_NAME": self.settings.contact_citizenship_field}},
mutating=False,
)
fields = result.result if isinstance(result.result, list) else []
entries = fields[0].get("LIST", []) if fields else []
async with self.repository.transaction() as connection:
for entry in entries:
if entry.get("ID") is None or not isinstance(entry.get("VALUE"), str):
continue
await connection.execute(
text(
"""
INSERT INTO bitrix_sync.citizenship_dictionary
(enum_id,display_value,loaded_at,expires_at)
VALUES (:id,:value,now(),now()+interval '1 hour')
ON CONFLICT (enum_id) DO UPDATE
SET display_value=excluded.display_value,loaded_at=now(),
expires_at=excluded.expires_at
"""
),
{"id": str(entry["ID"]), "value": entry["VALUE"]},
)
match = next(
(
entry["VALUE"]
for entry in entries
if str(entry.get("ID")) == str(enum_id)
),
None,
)
if match is None:
raise BusinessConflict("unknown_citizenship_enum", [str(enum_id)])
return match
async def _write_identity(
self, workflow_id: uuid.UUID, external_id: str, user_id: uuid.UUID, *, active: bool
) -> None:
fields: dict[str, Any] = {self.settings.contact_registered_field: "1" if active else "0"}
fields[self.settings.contact_user_id_field] = str(user_id) if active else ""
await self._command(
workflow_id,
"contact_update",
"crm.contact.update",
{"id": external_id, "fields": fields},
mutating=True,
)
async def _command(
self,
workflow_id: uuid.UUID,
command_type: str,
method: str,
params: dict[str, Any],
*,
mutating: bool,
) -> CrmResult:
command_id = uuid.uuid4()
async with self.repository.transaction() as connection:
await connection.execute(
text(
"""
INSERT INTO bitrix_sync.crm_commands
(id,workflow_id,command_type,safe_request,status,attempt_count,
next_attempt_at,created_at,updated_at)
VALUES (:id,:workflow_id,:type,:safe_request,'in_flight',1,now(),now(),now())
"""
),
{
"id": command_id,
"workflow_id": workflow_id,
"type": command_type,
"safe_request": {"keys": sorted(params), "method_class": method.split(".")[-1]},
},
)
while True:
limiter_delay = await self.repository.reserve_limiter_token(
self.settings.limiter_refill_per_sec, self.settings.limiter_burst
)
if limiter_delay <= 0:
break
await asyncio.sleep(limiter_delay)
async with self._in_flight:
result = await self.crm.call(method, params, mutating=mutating)
status = result.outcome.value
async with self.repository.transaction() as connection:
await connection.execute(
text(
"""
UPDATE bitrix_sync.crm_commands
SET status=:status,safe_error_code=:error,http_status=:http_status,
safe_response=:safe_response,
completed_at=CASE WHEN :terminal THEN now() END,
updated_at=now()
WHERE id=:id
"""
),
{
"id": command_id,
"status": status,
"error": result.error_code,
"http_status": result.http_status,
"safe_response": {"has_result": result.result is not None},
"terminal": result.outcome in {CrmOutcome.SUCCEEDED, CrmOutcome.PERMANENT},
},
)
if result.outcome == CrmOutcome.SUCCEEDED:
return result
if result.outcome == CrmOutcome.PERMANENT:
await self._technical_failure(workflow_id, result.error_code or "crm_permanent")
raise BusinessConflict("technical_configuration_failure")
raise RetryableWorkflow(result.error_code or result.outcome.value, result.retry_after)
async def _activate_mapping(
self, workflow_id: uuid.UUID, user_id: uuid.UUID, external_id: str
) -> None:
async with self.repository.transaction() as connection:
await connection.execute(
text(
"""
INSERT INTO bitrix_sync.entity_external_mapping
(id,entity_type,entity_id,external_system,external_entity_type,
external_id,status,opened_at,workflow_id,created_at,updated_at)
VALUES (gen_random_uuid(),'contact',:user_id,'bitrix24','contact',
:external_id,'active',now(),:workflow_id,now(),now())
ON CONFLICT DO NOTHING
"""
),
{
"workflow_id": workflow_id,
"user_id": user_id,
"external_id": external_id,
},
)
async def _coalesce_map_or_create(self, user_id: uuid.UUID) -> None:
async with self.repository.transaction() as connection:
await connection.execute(
text(
"""
INSERT INTO han_app.sync_queue
(id,task_type,entity_type,entity_id,dedup_key,payload_json,status,
attempt_count,next_attempt_at,created_at,updated_at)
VALUES (gen_random_uuid(),'contact.map_or_create','contact',:user_id,
'contact.map_or_create:'||:user_id::text,
jsonb_build_object('schema_version',1,'user_id',:user_id),
'pending',0,now(),now(),now())
ON CONFLICT DO NOTHING
"""
),
{"user_id": user_id},
)
async def _alert(
self, workflow_id: uuid.UUID, user_id: uuid.UUID, alert_type: str, candidates: list[str]
) -> None:
fingerprint = f"{alert_type}:{user_id}"
async with self.repository.transaction() as connection:
await connection.execute(
text(
"""
INSERT INTO bitrix_sync.business_alerts
(id,fingerprint,alert_type,severity,app_user_id,candidate_external_ids,
workflow_id,status,occurrence_count,first_occurred_at,last_occurred_at,
created_at,updated_at)
VALUES (gen_random_uuid(),encode(digest(:fingerprint,'sha256'),'hex'),
:type,'warning',:user_id,:candidates,:workflow_id,'open',1,
now(),now(),now(),now())
ON CONFLICT (alert_type,fingerprint) WHERE status='open'
DO UPDATE SET occurrence_count=business_alerts.occurrence_count+1,
last_occurred_at=now(),updated_at=now()
"""
),
{
"fingerprint": fingerprint,
"type": alert_type,
"user_id": user_id,
"candidates": candidates,
"workflow_id": workflow_id,
},
)
async def _manual(self, workflow_id: uuid.UUID, code: str, user_id: uuid.UUID) -> None:
async with self.repository.transaction() as connection:
await connection.execute(
text(
"""
UPDATE bitrix_sync.workflow_instances
SET state='waiting_manual',outcome=:code,updated_at=now()
WHERE id=:id
"""
),
{"id": workflow_id, "code": code},
)
async def _technical_failure(self, workflow_id: uuid.UUID, code: str) -> None:
async with self.repository.transaction() as connection:
await connection.execute(
text(
"""
INSERT INTO bitrix_sync.technical_dead_letters
(id,workflow_id,operation,safe_error_code,failed_at,created_at)
VALUES (gen_random_uuid(),:workflow_id,'crm_command',:code,now(),now())
"""
),
{"workflow_id": workflow_id, "code": code[:64]},
)
def _select_fields(self) -> list[str]:
return [
"ID",
"NAME",
"PHONE",
"EMAIL",
"CREATED_TIME",
"DATE_MODIFY",
self.settings.contact_user_id_field,
self.settings.contact_registered_field,
self.settings.contact_citizenship_field,
]
class RetryableWorkflow(Exception):
def __init__(self, code: str, retry_after: float | None = None) -> None:
self.code = code
self.retry_after = retry_after
super().__init__(code)
def _contact_ids(result: Any) -> list[str]:
if isinstance(result, dict):
values = result.get("CONTACT", [])
else:
values = result or []
return sorted({str(value) for value in values if str(value).isdigit()}, key=int)
@@ -0,0 +1,173 @@
from __future__ import annotations
import hmac
from contextlib import asynccontextmanager
from typing import Annotated
import uvicorn
from fastapi import Depends, FastAPI, Header, HTTPException, Query, Request, Response
from fastapi.responses import JSONResponse
from sqlalchemy import text
from app.config import Settings, load_settings
from app.repository import Repository
from app.security import WebhookValidationError, parse_bounded_form, validate_webhook
@asynccontextmanager
async def lifespan(app: FastAPI):
settings = load_settings()
app.state.settings = settings
app.state.repository = (
Repository(settings.database_url.get_secret_value(), settings.db_pool_size)
if settings.enabled and settings.database_url
else None
)
yield
if app.state.repository:
await app.state.repository.close()
app = FastAPI(
title="HAN Bitrix Sync",
version="0.1.0",
docs_url=None,
redoc_url=None,
lifespan=lifespan,
)
def settings(request: Request) -> Settings:
return request.app.state.settings
def repository(request: Request) -> Repository:
repo = request.app.state.repository
if repo is None:
raise HTTPException(status_code=503, detail="sync_disabled")
return repo
async def require_service_token(
request: Request,
authorization: Annotated[str | None, Header()] = None,
) -> None:
configured = settings(request).service_token
expected = f"Bearer {configured.get_secret_value()}" if configured else ""
if not authorization or not hmac.compare_digest(authorization, expected):
raise HTTPException(status_code=401, detail="unauthorized")
@app.get("/health/live", include_in_schema=True)
async def live() -> dict[str, str]:
return {"status": "live"}
@app.get("/health/ready", include_in_schema=True)
async def ready(request: Request) -> Response:
config = settings(request)
if not config.enabled:
return JSONResponse(
status_code=503,
content={"status": "not_ready", "reason": "sync_disabled"},
)
repo = repository(request)
if not await repo.ping():
return JSONResponse(status_code=503, content={"status": "not_ready", "reason": "database"})
return JSONResponse({"status": "ready", "mode": config.mode})
@app.get(
"/internal/sync/v1/status",
dependencies=[Depends(require_service_token)],
include_in_schema=True,
)
async def sync_status(request: Request) -> dict:
result = await repository(request).status()
result["mode"] = settings(request).mode
return result
@app.post("/bitrix/sync/webhook/contact", status_code=202, include_in_schema=True)
async def contact_webhook(
request: Request,
token: Annotated[str | None, Query(max_length=256)] = None,
ID: Annotated[str | None, Query(pattern=r"^[1-9][0-9]{0,19}$")] = None, # noqa: N803
) -> Response:
return await _receive(request, "contact", {"token": token or "", "ID": ID or ""})
@app.post("/bitrix/sync/webhook/alert", status_code=202, include_in_schema=True)
async def alert_webhook(
request: Request,
token: Annotated[str | None, Query(max_length=256)] = None,
ID: Annotated[str | None, Query(pattern=r"^[1-9][0-9]{0,19}$")] = None, # noqa: N803
) -> Response:
return await _receive(request, "alert", {"token": token or "", "ID": ID or ""})
async def _receive(request: Request, receiver: str, query: dict[str, str]) -> Response:
config = settings(request)
if not config.enabled:
raise HTTPException(status_code=503, detail="sync_disabled")
if request.headers.get("content-type", "").split(";", 1)[0].lower() != (
"application/x-www-form-urlencoded"
):
raise HTTPException(status_code=400, detail="invalid_content_type")
content_length = request.headers.get("content-length")
if content_length and (
not content_length.isdigit() or int(content_length) > config.webhook_max_body_bytes
):
raise HTTPException(status_code=413, detail="body_too_large")
body = await request.body()
if len(body) > config.webhook_max_body_bytes:
raise HTTPException(status_code=413, detail="body_too_large")
form = parse_bounded_form(body, max_fields=config.webhook_max_fields)
# The container is reachable only from the trusted VM2 nginx network.
# nginx overwrites X-Real-IP from the TCP peer after its CIDR check.
source_ip = request.headers.get("x-real-ip") or (request.client.host if request.client else "")
alert_entity_type_id = (
await _alert_entity_type(repository(request)) if receiver == "alert" else None
)
try:
event = validate_webhook(
receiver,
query,
form,
source_ip,
config,
alert_entity_type_id=alert_entity_type_id,
)
except PermissionError as exc:
raise HTTPException(status_code=403, detail="forbidden") from exc
except WebhookValidationError as exc:
raise HTTPException(status_code=400, detail="malformed_webhook") from exc
await repository(request).insert_webhook(
event.receiver_type, event.event_type, event.entity_id, event.source_ip
)
return Response(status_code=202)
async def _alert_entity_type(repo: Repository) -> int | None:
async with repo.engine.connect() as connection:
value = (
await connection.execute(
text(
"""
SELECT (value_json->>'entity_type_id')::integer
FROM bitrix_sync.settings
WHERE key='business_alerts' AND active=true AND validation_status='valid'
"""
)
)
).scalar_one_or_none()
return value
def run() -> None:
uvicorn.run(
"app.main:app",
host="0.0.0.0", # noqa: S104 - container-only port, not host-published
port=8080,
proxy_headers=False,
)
@@ -0,0 +1,56 @@
from __future__ import annotations
from dataclasses import dataclass
from datetime import UTC, datetime, timedelta
from typing import Any
from app.domain import select_email
class UnknownCitizenship(ValueError):
pass
@dataclass(frozen=True)
class CitizenshipEntry:
enum_id: str
display_value: str
loaded_at: datetime
class CitizenshipDictionary:
def __init__(self, ttl_seconds: int = 3600) -> None:
self.ttl = timedelta(seconds=ttl_seconds)
self._entries: dict[str, CitizenshipEntry] = {}
self.loaded_at: datetime | None = None
def load(self, values: list[dict[str, Any]], now: datetime | None = None) -> None:
loaded_at = now or datetime.now(UTC)
self._entries = {
str(item["ID"]): CitizenshipEntry(
str(item["ID"]), str(item["VALUE"]), loaded_at
)
for item in values
if item.get("ID") is not None and isinstance(item.get("VALUE"), str)
}
self.loaded_at = loaded_at
def resolve(self, enum_id: str | int | None, now: datetime | None = None) -> str | None:
if enum_id in (None, ""):
return None
entry = self._entries.get(str(enum_id))
if entry is None:
raise UnknownCitizenship(str(enum_id))
if (now or datetime.now(UTC)) - entry.loaded_at > self.ttl:
raise UnknownCitizenship(str(enum_id))
return entry.display_value
def crm_master_projection(
contact: dict[str, Any], citizenship: CitizenshipDictionary, citizenship_field: str
) -> dict[str, str | None]:
return {
"full_name": contact.get("NAME") or None,
"email": select_email(contact.get("EMAIL") or []),
"citizenship": citizenship.resolve(contact.get(citizenship_field)),
}
@@ -0,0 +1,139 @@
from __future__ import annotations
import asyncio
from datetime import UTC, datetime, timedelta
from typing import Any
from sqlalchemy import text
from app.config import load_settings
from app.crm import CrmClient, CrmOutcome
from app.repository import Repository
class IncrementalReconciler:
def __init__(self, repository: Repository, crm: CrmClient, registered_rest_field: str) -> None:
self.repository = repository
self.crm = crm
self.registered_rest_field = registered_rest_field
async def run_once(self, overlap_seconds: int) -> int:
async with self.repository.transaction() as connection:
acquired = (
await connection.execute(
text(
"""
SELECT pg_try_advisory_xact_lock(
hashtext('bitrix-contact-reconciliation')
)
"""
)
)
).scalar_one()
if not acquired:
return 0
cursor = (
await connection.execute(
text(
"""
SELECT watermark FROM bitrix_sync.reconciliation_cursors
WHERE job_type='contact_incremental' FOR UPDATE
"""
)
)
).scalar_one_or_none()
started_at = datetime.now(UTC)
since = (cursor or datetime(1970, 1, 1, tzinfo=UTC)) - timedelta(seconds=overlap_seconds)
start = 0
scanned: list[str] = []
while True:
result = await self.crm.call(
"crm.item.list",
{
"entityTypeId": 3,
"select": ["id"],
"filter": {
">=updatedTime": since.replace(tzinfo=None).isoformat(timespec="seconds"),
"opened": 1,
self.registered_rest_field: 1,
},
"start": start,
},
mutating=False,
)
if result.outcome != CrmOutcome.SUCCEEDED:
raise RuntimeError(result.error_code or "reconciliation_failed")
payload: dict[str, Any] = result.result or {}
items = payload.get("items", payload if isinstance(payload, list) else [])
scanned.extend(str(item["id"]) for item in items if "id" in item)
next_start = payload.get("next")
if next_start is None:
break
start = int(next_start)
await self._enqueue_changed(scanned)
async with self.repository.transaction() as connection:
await connection.execute(
text(
"""
INSERT INTO bitrix_sync.reconciliation_cursors
(job_type,watermark,overlap_seconds,last_success_at,last_scanned_count,
created_at,updated_at)
VALUES ('contact_incremental',:watermark,:overlap,now(),:count,now(),now())
ON CONFLICT (job_type) DO UPDATE
SET watermark=excluded.watermark,overlap_seconds=excluded.overlap_seconds,
last_success_at=now(),last_scanned_count=excluded.last_scanned_count,
updated_at=now()
"""
),
{"watermark": started_at, "overlap": overlap_seconds, "count": len(scanned)},
)
return len(scanned)
async def _enqueue_changed(self, external_ids: list[str]) -> None:
if not external_ids:
return
async with self.repository.transaction() as connection:
await connection.execute(
text(
"""
INSERT INTO bitrix_sync.webhook_inbox
(id,receiver_type,event_type,external_entity_id,status,coalesced_count,
received_at,last_received_at)
SELECT gen_random_uuid(),'contact','contact.reconciliation',id,'received',1,
now(),now()
FROM unnest(CAST(:ids AS text[])) id
WHERE NOT EXISTS (
SELECT 1 FROM bitrix_sync.webhook_inbox w
WHERE w.receiver_type='contact' AND w.external_entity_id=id
AND w.status IN ('received','processing','retry_wait')
)
"""
),
{"ids": external_ids},
)
async def reconciliation_main() -> None:
settings = load_settings()
if not settings.enabled:
return
assert settings.database_url and settings.crm_rest_webhook_url and settings.portal_host
assert settings.contact_registered_field
repository = Repository(settings.database_url.get_secret_value(), settings.db_pool_size)
crm = CrmClient(
settings.crm_rest_webhook_url.get_secret_value(),
settings.portal_host,
settings.http_timeout_sec,
)
reconciler = IncrementalReconciler(
repository, crm, settings.rest_field_name(settings.contact_registered_field)
)
try:
await reconciler.run_once(settings.reconciliation_overlap_seconds)
finally:
await crm.close()
await repository.close()
def run() -> None:
asyncio.run(reconciliation_main())
@@ -0,0 +1,533 @@
from __future__ import annotations
import os
import ssl
import uuid
from collections.abc import AsyncIterator
from contextlib import asynccontextmanager
from dataclasses import dataclass
from datetime import UTC, datetime
from typing import Any
from sqlalchemy import text
from sqlalchemy.ext.asyncio import AsyncConnection, AsyncEngine, create_async_engine
def postgres_ssl_context() -> ssl.SSLContext:
ca_file = os.environ.get("PG_CA_FILE")
if not ca_file:
raise RuntimeError("PG_CA_FILE is required")
context = ssl.create_default_context(cafile=ca_file)
context.check_hostname = True
context.verify_mode = ssl.CERT_REQUIRED
return context
@dataclass(frozen=True)
class LeasedTask:
id: uuid.UUID
task_type: str
user_id: uuid.UUID
lease_token: uuid.UUID
attempt_count: int
@dataclass(frozen=True)
class LeasedWebhook:
id: uuid.UUID
receiver_type: str
event_type: str
external_id: str
lease_token: uuid.UUID
attempt_count: int
@dataclass(frozen=True)
class Profile:
user_id: uuid.UUID
phone: str
identity_status: str
profile_status: str
class Repository:
def __init__(self, database_url: str, pool_size: int = 5) -> None:
self.engine: AsyncEngine = create_async_engine(
database_url,
pool_size=pool_size,
pool_pre_ping=True,
connect_args={"ssl": postgres_ssl_context()},
)
async def close(self) -> None:
await self.engine.dispose()
@asynccontextmanager
async def transaction(self) -> AsyncIterator[AsyncConnection]:
async with self.engine.begin() as connection:
yield connection
async def ping(self) -> bool:
try:
async with self.engine.connect() as connection:
await connection.execute(text("SELECT 1"))
return True
except Exception:
return False
async def claim_tasks(
self, worker_id: str, limit: int, lease_seconds: int
) -> list[LeasedTask]:
sql = text(
"""
WITH candidates AS (
SELECT id
FROM han_app.sync_queue
WHERE status IN ('pending','retry_wait')
AND next_attempt_at <= now()
AND (locked_until IS NULL OR locked_until < now())
ORDER BY next_attempt_at, created_at
FOR UPDATE SKIP LOCKED
LIMIT :limit
)
UPDATE han_app.sync_queue q
SET status='leased', locked_by=:worker_id,
locked_until=now() + make_interval(secs => :lease_seconds),
lease_token=gen_random_uuid(), updated_at=now()
FROM candidates c
WHERE q.id=c.id
RETURNING q.id, q.task_type, q.entity_id, q.lease_token, q.attempt_count
"""
)
async with self.engine.begin() as connection:
rows = (
await connection.execute(
sql,
{"worker_id": worker_id, "limit": limit, "lease_seconds": lease_seconds},
)
).mappings()
return [
LeasedTask(
row["id"],
row["task_type"],
row["entity_id"],
row["lease_token"],
row["attempt_count"],
)
for row in rows
]
async def load_profile(self, user_id: uuid.UUID) -> Profile | None:
sql = text(
"""
SELECT i.id user_id, i.phone_number phone, i.record_status identity_status,
p.record_status profile_status
FROM han_app.user_identities i
JOIN han_app.client_profiles p ON p.user_id=i.id
WHERE i.id=:user_id
"""
)
async with self.engine.connect() as connection:
row = (await connection.execute(sql, {"user_id": user_id})).mappings().first()
return Profile(**row) if row else None
async def active_mapping(self, user_id: uuid.UUID) -> str | None:
sql = text(
"""
SELECT external_id FROM bitrix_sync.entity_external_mapping
WHERE external_system='bitrix24' AND entity_type='contact'
AND entity_id=:user_id AND status='active'
"""
)
async with self.engine.connect() as connection:
return (await connection.execute(sql, {"user_id": user_id})).scalar_one_or_none()
async def create_workflow(self, task: LeasedTask) -> uuid.UUID:
workflow_id = uuid.uuid4()
sql = text(
"""
INSERT INTO bitrix_sync.workflow_instances
(id, workflow_type, user_id, state, current_step, source_task_id,
deadline_at, created_at, updated_at)
VALUES (:id, :workflow_type, :user_id, 'created', 'load_profile', :task_id,
now() + interval '24 hours', now(), now())
ON CONFLICT (source_task_id) DO UPDATE SET updated_at=now()
RETURNING id
"""
)
async with self.engine.begin() as connection:
return (
await connection.execute(
sql,
{
"id": workflow_id,
"workflow_type": task.task_type,
"user_id": task.user_id,
"task_id": task.id,
},
)
).scalar_one()
async def complete_task(self, task: LeasedTask, workflow_id: uuid.UUID) -> bool:
async with self.engine.begin() as connection:
result = await connection.execute(
text(
"""
UPDATE han_app.sync_queue
SET status='processed', completed_at=now(), locked_by=NULL,
locked_until=NULL, lease_token=NULL, updated_at=now()
WHERE id=:id AND status='leased' AND lease_token=:lease_token
"""
),
{"id": task.id, "lease_token": task.lease_token},
)
if result.rowcount != 1:
return False
await connection.execute(
text(
"""
UPDATE bitrix_sync.workflow_instances
SET state='succeeded', current_step='done', outcome='processed',
completed_at=now(), updated_at=now()
WHERE id=:workflow_id AND state NOT IN ('succeeded','failed','cancelled')
"""
),
{"workflow_id": workflow_id},
)
return True
async def retry_task(
self, task: LeasedTask, safe_code: str, delay_seconds: float
) -> bool:
sql = text(
"""
UPDATE han_app.sync_queue
SET status='retry_wait', attempt_count=attempt_count+1,
next_attempt_at=now() + make_interval(secs => :delay),
last_error_code=:code, last_error_at=now(),
locked_by=NULL, locked_until=NULL, lease_token=NULL, updated_at=now()
WHERE id=:id AND status='leased' AND lease_token=:lease_token
"""
)
async with self.engine.begin() as connection:
result = await connection.execute(
sql,
{
"id": task.id,
"lease_token": task.lease_token,
"code": safe_code[:64],
"delay": delay_seconds,
},
)
return result.rowcount == 1
async def insert_webhook(
self,
receiver_type: str,
event_type: str,
entity_id: str,
source_ip: str,
) -> uuid.UUID:
inbox_id = uuid.uuid4()
async with self.engine.begin() as connection:
existing = await connection.execute(
text(
"""
SELECT id FROM bitrix_sync.webhook_inbox
WHERE receiver_type=:receiver AND external_entity_id=:entity_id
AND status IN ('received','processing','retry_wait')
FOR UPDATE
"""
),
{"receiver": receiver_type, "entity_id": entity_id},
)
if row := existing.first():
await connection.execute(
text(
"""
UPDATE bitrix_sync.webhook_inbox
SET coalesced_count=coalesced_count+1, last_received_at=now()
WHERE id=:id
"""
),
{"id": row[0]},
)
return row[0]
await connection.execute(
text(
"""
INSERT INTO bitrix_sync.webhook_inbox
(id, receiver_type, event_type, external_entity_id, source_ip,
status, coalesced_count, received_at, last_received_at)
VALUES (:id,:receiver,:event,:entity_id,CAST(:source_ip AS inet),
'received',1,now(),now())
"""
),
{
"id": inbox_id,
"receiver": receiver_type,
"event": event_type,
"entity_id": entity_id,
"source_ip": source_ip,
},
)
return inbox_id
async def claim_webhooks(
self, worker_id: str, limit: int, lease_seconds: int
) -> list[LeasedWebhook]:
sql = text(
"""
WITH candidates AS (
SELECT id FROM bitrix_sync.webhook_inbox
WHERE status IN ('received','retry_wait') AND next_attempt_at<=now()
AND (locked_until IS NULL OR locked_until<now())
ORDER BY next_attempt_at,received_at
FOR UPDATE SKIP LOCKED LIMIT :limit
)
UPDATE bitrix_sync.webhook_inbox w
SET status='processing',locked_by=:worker_id,
locked_until=now()+make_interval(secs=>:lease_seconds),
lease_token=gen_random_uuid()
FROM candidates c WHERE w.id=c.id
RETURNING w.id,w.receiver_type,w.event_type,w.external_entity_id,
w.lease_token,w.attempt_count
"""
)
async with self.engine.begin() as connection:
rows = (
await connection.execute(
sql,
{"worker_id": worker_id, "limit": limit, "lease_seconds": lease_seconds},
)
).mappings()
return [
LeasedWebhook(
row["id"],
row["receiver_type"],
row["event_type"],
row["external_entity_id"],
row["lease_token"],
row["attempt_count"],
)
for row in rows
]
async def mapped_user_for_external(self, external_id: str) -> uuid.UUID | None:
async with self.engine.connect() as connection:
return (
await connection.execute(
text(
"""
SELECT entity_id FROM bitrix_sync.entity_external_mapping
WHERE external_system='bitrix24' AND external_entity_type='contact'
AND external_id=:external_id AND status='active'
"""
),
{"external_id": external_id},
)
).scalar_one_or_none()
async def apply_crm_profile(
self,
user_id: uuid.UUID,
external_id: str,
*,
full_name: str | None,
citizenship: str | None,
email: str | None,
source_updated_at: datetime | None,
source: str,
) -> None:
async with self.engine.begin() as connection:
await connection.execute(text("SET LOCAL han.sync_suppress='true'"))
await connection.execute(
text(
"""
UPDATE han_app.client_profiles
SET full_name=:full_name,citizenship=:citizenship,email=:email,updated_at=now()
WHERE user_id=:user_id AND record_status='A'
"""
),
{
"user_id": user_id,
"full_name": full_name,
"citizenship": citizenship,
"email": email,
},
)
await connection.execute(
text(
"""
INSERT INTO bitrix_sync.contact_snapshots
(id,mapping_id,user_id,external_id,full_name_hash,email_hash,
citizenship_hash,source_updated_at,last_applied_source,
last_webhook_received_at,created_at,updated_at)
SELECT gen_random_uuid(),m.id,:user_id,:external_id,
encode(digest(coalesce(:full_name,''),'sha256'),'hex'),
encode(digest(coalesce(:email,''),'sha256'),'hex'),
encode(digest(coalesce(:citizenship,''),'sha256'),'hex'),
:source_updated_at,:source,
CASE WHEN :source='webhook' THEN now() END,now(),now()
FROM bitrix_sync.entity_external_mapping m
WHERE m.entity_id=:user_id AND m.external_id=:external_id AND m.status='active'
ON CONFLICT (mapping_id) DO UPDATE
SET full_name_hash=excluded.full_name_hash,email_hash=excluded.email_hash,
citizenship_hash=excluded.citizenship_hash,
source_updated_at=excluded.source_updated_at,
last_applied_source=excluded.last_applied_source,
last_webhook_received_at=coalesce(
excluded.last_webhook_received_at,
bitrix_sync.contact_snapshots.last_webhook_received_at),
updated_at=now()
"""
),
{
"user_id": user_id,
"external_id": external_id,
"full_name": full_name,
"citizenship": citizenship,
"email": email,
"source_updated_at": source_updated_at,
"source": source,
},
)
async def complete_webhook(self, item: LeasedWebhook) -> bool:
async with self.engine.begin() as connection:
result = await connection.execute(
text(
"""
UPDATE bitrix_sync.webhook_inbox
SET status='processed',processed_at=now(),locked_by=NULL,
locked_until=NULL,lease_token=NULL
WHERE id=:id AND status='processing' AND lease_token=:lease_token
"""
),
{"id": item.id, "lease_token": item.lease_token},
)
return result.rowcount == 1
async def retry_webhook(
self, item: LeasedWebhook, safe_code: str, delay_seconds: float
) -> bool:
async with self.engine.begin() as connection:
result = await connection.execute(
text(
"""
UPDATE bitrix_sync.webhook_inbox
SET status='retry_wait',attempt_count=attempt_count+1,
next_attempt_at=now()+make_interval(secs=>:delay),
safe_error_code=:code,locked_by=NULL,locked_until=NULL,lease_token=NULL
WHERE id=:id AND status='processing' AND lease_token=:lease_token
"""
),
{
"id": item.id,
"lease_token": item.lease_token,
"code": safe_code[:64],
"delay": delay_seconds,
},
)
return result.rowcount == 1
async def pending_rebind_ids(self, limit: int) -> list[uuid.UUID]:
async with self.engine.connect() as connection:
rows = await connection.execute(
text(
"""
SELECT id FROM bitrix_sync.rebind_requests
WHERE status IN ('pending','retry_wait')
ORDER BY requested_at LIMIT :limit
"""
),
{"limit": limit},
)
return list(rows.scalars())
async def reserve_limiter_token(self, refill_per_second: float, burst: int) -> float:
async with self.engine.begin() as connection:
row = (
await connection.execute(
text(
"""
SELECT tokens,capacity,refill_per_second,
extract(epoch FROM now()-updated_at) elapsed,
greatest(0,extract(epoch FROM blocked_until-now())) blocked
FROM bitrix_sync.limiter_coordination
WHERE limiter_key='bitrix24:portal'
FOR UPDATE
"""
)
)
).mappings().first()
if row is None:
await connection.execute(
text(
"""
INSERT INTO bitrix_sync.limiter_coordination
(limiter_key,tokens,capacity,refill_per_second,updated_at,fencing_token)
VALUES ('bitrix24:portal',:tokens,:capacity,:refill,now(),1)
"""
),
{"tokens": max(0, burst - 1), "capacity": burst, "refill": refill_per_second},
)
return 0
blocked = float(row["blocked"] or 0)
tokens = min(
float(burst),
float(row["tokens"]) + float(row["elapsed"] or 0) * refill_per_second,
)
if blocked > 0:
delay = blocked
elif tokens >= 1:
tokens -= 1
delay = 0
else:
delay = (1 - tokens) / refill_per_second
await connection.execute(
text(
"""
UPDATE bitrix_sync.limiter_coordination
SET tokens=:tokens,capacity=:capacity,refill_per_second=:refill,
updated_at=now(),fencing_token=fencing_token+1
WHERE limiter_key='bitrix24:portal'
"""
),
{
"tokens": tokens,
"capacity": burst,
"refill": refill_per_second,
},
)
return delay
async def status(self) -> dict[str, Any]:
queries = {
"queue": "SELECT status, count(*) count FROM han_app.sync_queue GROUP BY status",
"workflows": (
"SELECT state, count(*) count FROM bitrix_sync.workflow_instances GROUP BY state"
),
"commands": (
"SELECT status, count(*) count "
"FROM bitrix_sync.crm_commands GROUP BY status"
),
"webhook_lag_seconds": (
"SELECT coalesce(extract(epoch from now()-min(received_at)),0) "
"FROM bitrix_sync.webhook_inbox WHERE status IN ('received','retry_wait')"
),
"settings_version": (
"SELECT version FROM bitrix_sync.settings_versions "
"WHERE active=true AND validation_status='valid' ORDER BY activated_at DESC LIMIT 1"
),
}
output: dict[str, Any] = {}
async with self.engine.connect() as connection:
for key, sql in queries.items():
result = await connection.execute(text(sql))
if key in {"queue", "workflows", "commands"}:
output[key] = {row.status: row.count for row in result}
else:
output[key] = result.scalar_one_or_none()
output["generated_at"] = datetime.now(UTC).isoformat()
return output
@@ -0,0 +1,120 @@
from __future__ import annotations
import hmac
import ipaddress
import re
from collections.abc import Mapping
from dataclasses import dataclass
from urllib.parse import parse_qsl
from app.config import Settings
SECRET_PATTERNS = (
re.compile(r"(?i)(token|authorization|password|secret)=([^&\s]+)"),
re.compile(r"https://[^/\s]+/rest/[0-9]+/[^/\s]+/"),
)
def token_matches(received: str | None, current: str, previous: str | None = None) -> bool:
candidate = (received or "").encode()
current_match = hmac.compare_digest(candidate, current.encode())
previous_match = hmac.compare_digest(candidate, (previous or "").encode())
return current_match or (previous is not None and previous_match)
def redact(value: object) -> str:
text = str(value)
for pattern in SECRET_PATTERNS:
text = pattern.sub(
lambda match: (
f"{match.group(1)}=[REDACTED]"
if match.lastindex == 2
else "https://[REDACTED]/"
),
text,
)
if "@" in text or re.search(r"\+7[0-9]{10}", text):
return "[PII_REDACTED]"
return text[:512]
@dataclass(frozen=True)
class WebhookEvent:
receiver_type: str
entity_id: str
event_type: str
source_ip: str
class WebhookValidationError(ValueError):
pass
def parse_bounded_form(body: bytes, *, max_fields: int) -> dict[str, str]:
try:
pairs = parse_qsl(body.decode("utf-8"), keep_blank_values=True, max_num_fields=max_fields)
except (UnicodeDecodeError, ValueError) as exc:
raise WebhookValidationError("malformed form") from exc
if len(pairs) > max_fields:
raise WebhookValidationError("too many form fields")
data: dict[str, str] = {}
for key, value in pairs:
if len(key) > 128 or len(value) > 512:
raise WebhookValidationError("form field too long")
data[key] = value
return data
def validate_webhook(
receiver: str,
query: Mapping[str, str],
form: Mapping[str, str],
source_ip: str,
settings: Settings,
*,
alert_entity_type_id: int | None,
) -> WebhookEvent:
try:
ip = ipaddress.ip_address(source_ip)
except ValueError as exc:
raise WebhookValidationError("invalid source address") from exc
if not any(ip in network for network in settings.allowed_networks):
raise PermissionError("source_ip")
configured = (
settings.contact_receiver_token if receiver == "contact" else settings.alert_receiver_token
)
previous = (
settings.contact_receiver_previous_token
if receiver == "contact"
else settings.alert_receiver_previous_token
)
if not configured or not token_matches(
query.get("token"),
configured.get_secret_value(),
previous.get_secret_value() if previous else None,
):
raise PermissionError("token")
if form.get("auth[domain]", "").lower() != str(settings.portal_host).lower():
raise WebhookValidationError("portal mismatch")
if form.get("auth[member_id]") != settings.portal_member_id:
raise WebhookValidationError("member mismatch")
if form.get("document_id[0]") != "crm":
raise WebhookValidationError("invalid document module")
document_type = form.get("document_id[1]")
document_id = form.get("document_id[2]", "")
if receiver == "contact":
match = re.fullmatch(r"CONTACT_([1-9][0-9]*)", document_id)
if document_type != "CCrmDocumentContact" or not match:
raise WebhookValidationError("invalid contact document")
else:
match = re.fullmatch(r"DYNAMIC_([1-9][0-9]*)_([1-9][0-9]*)", document_id)
if not match or not document_type or "Dynamic" not in document_type:
raise WebhookValidationError("invalid alert document")
if alert_entity_type_id is None or int(match.group(1)) != alert_entity_type_id:
raise WebhookValidationError("alert entity type mismatch")
entity_id = match.group(match.lastindex or 1)
if query.get("ID") != entity_id:
raise WebhookValidationError("query/document ID mismatch")
return WebhookEvent(receiver, entity_id, f"{receiver}.changed", source_ip)
@@ -0,0 +1,78 @@
from __future__ import annotations
import asyncio
import signal
import socket
import uuid
from app.config import load_settings
from app.crm import CrmClient
from app.domain import full_jitter_delay
from app.engine import RetryableWorkflow, WorkflowEngine
from app.repository import Repository
async def worker_main() -> None:
settings = load_settings()
if not settings.enabled:
return
assert settings.database_url and settings.crm_rest_webhook_url and settings.portal_host
repository = Repository(settings.database_url.get_secret_value(), settings.db_pool_size)
crm = CrmClient(
settings.crm_rest_webhook_url.get_secret_value(),
settings.portal_host,
settings.http_timeout_sec,
)
engine = WorkflowEngine(repository, crm, settings)
stop = asyncio.Event()
loop = asyncio.get_running_loop()
for event in (signal.SIGINT, signal.SIGTERM):
try:
loop.add_signal_handler(event, stop.set)
except NotImplementedError:
pass
worker_id = f"{socket.gethostname()}:{uuid.uuid4()}"
try:
while not stop.is_set():
tasks = await repository.claim_tasks(
worker_id, settings.claim_size, settings.lease_seconds
)
webhooks = await repository.claim_webhooks(
worker_id, settings.claim_size, settings.lease_seconds
)
rebind_ids = await repository.pending_rebind_ids(settings.claim_size)
if not tasks and not webhooks and not rebind_ids:
try:
await asyncio.wait_for(stop.wait(), timeout=1)
except TimeoutError:
continue
for task in tasks:
if stop.is_set():
break
await engine.process(task)
for item in webhooks:
if stop.is_set():
break
try:
await engine.process_webhook(item)
except RetryableWorkflow as exc:
delay = exc.retry_after or full_jitter_delay(
item.attempt_count + 1,
settings.retry_base_seconds,
settings.retry_max_seconds,
)
await repository.retry_webhook(item, exc.code, delay)
for request_id in rebind_ids:
if stop.is_set():
break
try:
await engine.process_rebind(request_id)
except RetryableWorkflow:
continue
finally:
await crm.close()
await repository.close()
def run() -> None:
asyncio.run(worker_main())
@@ -0,0 +1,83 @@
services:
bitrix-sync:
build: .
image: han-bitrix-sync:${BITRIX_SYNC_IMAGE_TAG:-local}
command: ["han-bitrix-sync-api"]
user: "10001:10001"
read_only: true
security_opt: ["no-new-privileges:true"]
cap_drop: ["ALL"]
tmpfs: ["/tmp:rw,noexec,nosuid,nodev,size=16m"]
expose: ["8080"]
environment: &sync_environment
BITRIX_SYNC_ENABLED: ${BITRIX_SYNC_ENABLED:-false}
BITRIX_SYNC_MODE: ${BITRIX_SYNC_MODE:-disabled}
BITRIX_SYNC_DATABASE_URL_FILE: /run/secrets/bitrix_sync_database_url
BITRIX_SYNC_CRM_REST_WEBHOOK_URL_FILE: /run/secrets/bitrix_sync_crm_url
BITRIX_SYNC_CONTACT_RECEIVER_TOKEN_FILE: /run/secrets/bitrix_sync_contact_token
BITRIX_SYNC_ALERT_RECEIVER_TOKEN_FILE: /run/secrets/bitrix_sync_alert_token
BITRIX_SYNC_SERVICE_TOKEN_FILE: /run/secrets/bitrix_sync_service_token
BITRIX_SYNC_PORTAL_HOST: ${BITRIX_SYNC_PORTAL_HOST:-}
BITRIX_SYNC_PORTAL_MEMBER_ID: ${BITRIX_SYNC_PORTAL_MEMBER_ID:-}
BITRIX_SYNC_PUBLIC_BASE_URL: ${BITRIX_SYNC_PUBLIC_BASE_URL:-}
BITRIX_SYNC_CONTACT_USER_ID_FIELD: ${BITRIX_SYNC_CONTACT_USER_ID_FIELD:-}
BITRIX_SYNC_CONTACT_REGISTERED_FIELD: ${BITRIX_SYNC_CONTACT_REGISTERED_FIELD:-}
BITRIX_SYNC_CONTACT_CITIZENSHIP_FIELD: ${BITRIX_SYNC_CONTACT_CITIZENSHIP_FIELD:-}
BITRIX_SYNC_WEBHOOK_ALLOWED_CIDRS: ${BITRIX_WEBHOOK_ALLOWED_CIDRS:-}
volumes: &sync_secrets
- /run/han-chat/secrets/bitrix-sync/database-url:/run/secrets/bitrix_sync_database_url:ro
- /run/han-chat/secrets/bitrix-sync/crm-rest-webhook-url:/run/secrets/bitrix_sync_crm_url:ro
- /run/han-chat/secrets/bitrix-sync/contact-receiver-token:/run/secrets/bitrix_sync_contact_token:ro
- /run/han-chat/secrets/bitrix-sync/alert-receiver-token:/run/secrets/bitrix_sync_alert_token:ro
- /run/han-chat/secrets/bitrix-sync/service-token:/run/secrets/bitrix_sync_service_token:ro
networks: [backend, egress, observability]
pids_limit: 128
mem_limit: 256m
cpus: 0.50
restart: unless-stopped
healthcheck:
test:
["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health/live',timeout=2)"]
interval: 30s
timeout: 3s
retries: 3
bitrix-sync-worker:
image: han-bitrix-sync:${BITRIX_SYNC_IMAGE_TAG:-local}
command: ["han-bitrix-sync-worker"]
user: "10001:10001"
read_only: true
security_opt: ["no-new-privileges:true"]
cap_drop: ["ALL"]
tmpfs: ["/tmp:rw,noexec,nosuid,nodev,size=16m"]
environment: *sync_environment
volumes: *sync_secrets
networks: [egress, observability]
pids_limit: 128
mem_limit: 256m
cpus: 0.75
restart: unless-stopped
bitrix-sync-reconciliation:
image: han-bitrix-sync:${BITRIX_SYNC_IMAGE_TAG:-local}
command: ["han-bitrix-sync-reconciliation"]
user: "10001:10001"
read_only: true
security_opt: ["no-new-privileges:true"]
cap_drop: ["ALL"]
tmpfs: ["/tmp:rw,noexec,nosuid,nodev,size=16m"]
environment: *sync_environment
volumes: *sync_secrets
networks: [egress, observability]
pids_limit: 128
mem_limit: 192m
cpus: 0.50
restart: "no"
networks:
backend:
external: true
egress:
external: true
observability:
external: true
@@ -0,0 +1,121 @@
openapi: 3.1.0
info:
title: HAN Bitrix Sync
version: 0.1.0
paths:
/health/live:
get:
operationId: healthLive
responses:
"200":
description: Process is alive
/health/ready:
get:
operationId: healthReady
responses:
"200":
description: Full-mode configuration and database are ready
"503":
description: Disabled or a core dependency is not ready
/internal/sync/v1/status:
get:
operationId: syncStatus
security:
- bearerAuth: []
responses:
"200":
description: Low-cardinality operational status without PII
"401":
description: Missing or invalid service token
/bitrix/sync/webhook/contact:
post:
operationId: receiveContactRobot
parameters:
- $ref: "#/components/parameters/ReceiverToken"
- $ref: "#/components/parameters/EntityId"
requestBody:
required: true
content:
application/x-www-form-urlencoded:
schema:
$ref: "#/components/schemas/RobotForm"
responses:
"202":
description: Event durably stored
"400":
description: Malformed robot contract
"403":
description: Receiver authentication rejected
"413":
description: Body too large
"503":
description: Sync is disabled
/bitrix/sync/webhook/alert:
post:
operationId: receiveAlertRobot
parameters:
- $ref: "#/components/parameters/ReceiverToken"
- $ref: "#/components/parameters/EntityId"
requestBody:
required: true
content:
application/x-www-form-urlencoded:
schema:
$ref: "#/components/schemas/RobotForm"
responses:
"202":
description: Event durably stored
"400":
description: Malformed robot contract
"403":
description: Receiver authentication rejected
"413":
description: Body too large
"503":
description: Sync is disabled
components:
securitySchemes:
bearerAuth:
type: http
scheme: bearer
parameters:
ReceiverToken:
name: token
in: query
required: true
schema:
type: string
maxLength: 256
description: Secret receiver token; MUST be excluded from logs and traces.
EntityId:
name: ID
in: query
required: true
schema:
type: string
pattern: "^[1-9][0-9]{0,19}$"
schemas:
RobotForm:
type: object
additionalProperties: false
required:
- document_id[0]
- document_id[1]
- document_id[2]
- auth[domain]
- auth[member_id]
properties:
document_id[0]:
type: string
document_id[1]:
type: string
document_id[2]:
type: string
auth[domain]:
type: string
auth[member_id]:
type: string
auth[client_endpoint]:
type: string
auth[server_endpoint]:
type: string
@@ -0,0 +1,50 @@
[project]
name = "han-bitrix-sync"
version = "0.1.0"
description = "Durable HAN App to Bitrix24 Contact synchronization"
requires-python = ">=3.12"
dependencies = [
"alembic>=1.16,<2",
"asyncpg>=0.30,<1",
"fastapi>=0.116,<1",
"httpx>=0.28,<1",
"pydantic-settings>=2.10,<3",
"python-multipart>=0.0.20,<1",
"sqlalchemy[asyncio]>=2.0.41,<3",
"structlog>=25,<26",
"uvicorn[standard]>=0.35,<1",
]
[project.optional-dependencies]
dev = [
"pytest>=8.4,<9",
"pytest-asyncio>=1.0,<2",
"ruff>=0.12,<1",
]
[project.scripts]
han-bitrix-sync-api = "app.main:run"
han-bitrix-sync-worker = "app.worker:run"
han-bitrix-sync-reconciliation = "app.reconciliation:run"
[build-system]
requires = ["hatchling"]
build-backend = "hatchling.build"
[tool.hatch.build.targets.wheel]
packages = ["app"]
[tool.pytest.ini_options]
asyncio_mode = "auto"
testpaths = ["tests"]
[tool.ruff]
target-version = "py312"
line-length = 100
[tool.ruff.lint]
select = ["E", "F", "I", "UP", "B", "ASYNC", "S"]
ignore = ["S101"]
[tool.ruff.lint.per-file-ignores]
"tests/**" = ["S106", "S311"]
@@ -0,0 +1,25 @@
from __future__ import annotations
import pytest
from app.config import Settings
@pytest.fixture
def full_settings() -> Settings:
return Settings(
enabled=True,
mode="full",
database_url="postgresql+asyncpg://user:pass@db/han",
crm_rest_webhook_url="https://portal.example/rest/1/credential/",
contact_receiver_token="contact-token-value-32-characters",
alert_receiver_token="alert-token-value-32-characters---",
service_token="service-token-value-32-characters-",
portal_host="portal.example",
portal_member_id="member_12345678",
public_base_url="https://sync.example",
contact_user_id_field="UF_CRM_100",
contact_registered_field="UF_CRM_101",
contact_citizenship_field="UF_CRM_102",
webhook_allowed_cidrs="203.0.113.0/24",
)
@@ -0,0 +1,77 @@
from __future__ import annotations
import ast
import re
from pathlib import Path
import pytest
from pydantic import ValidationError
from app.config import Settings
def test_disabled_mode_needs_no_secrets() -> None:
settings = Settings(enabled=False, mode="disabled")
assert settings.enabled is False
def test_full_mode_rejects_portal_host_mismatch() -> None:
with pytest.raises(ValidationError, match="approved portal host"):
Settings(
enabled=True,
mode="full",
database_url="postgresql+asyncpg://u:p@db/han",
crm_rest_webhook_url="https://evil.example/rest/1/token/",
contact_receiver_token="contact",
alert_receiver_token="alert",
service_token="service",
portal_host="portal.example",
portal_member_id="member_12345678",
public_base_url="https://sync.example",
contact_user_id_field="UF_CRM_1",
contact_registered_field="UF_CRM_2",
contact_citizenship_field="UF_CRM_3",
webhook_allowed_cidrs="203.0.113.0/24",
)
def test_rest_field_conversion_is_deterministic() -> None:
assert Settings.rest_field_name("UF_CRM_1778692456") == "ufCrm_1778692456"
with pytest.raises(ValueError):
Settings.rest_field_name("uf_crm_1")
def test_alembic_chain_preserves_legacy_baseline() -> None:
versions = Path(__file__).parents[1] / "alembic" / "versions"
revisions: dict[str, str | None] = {}
for migration in versions.glob("*.py"):
assignments = {
node.target.id: node.value.value
for node in ast.parse(migration.read_text(encoding="utf-8")).body
if isinstance(node, ast.AnnAssign)
and isinstance(node.target, ast.Name)
and node.target.id in {"revision", "down_revision"}
and isinstance(node.value, ast.Constant)
}
revisions[assignments["revision"]] = assignments["down_revision"]
assert revisions == {
"0001_sync_baseline": None,
"0001_bitrix_sync_full": "0001_sync_baseline",
"0002_app_queue_contract": "0001_bitrix_sync_full",
}
def test_app_queue_migration_does_not_revoke_foreign_schema_privileges() -> None:
migration = (
Path(__file__).parents[1]
/ "alembic"
/ "versions"
/ "0002_app_queue_contract.py"
)
source = migration.read_text(encoding="utf-8")
assert "FROM han_app.entity_external_mapping" in source
assert "REVOKE" not in source
assert re.search(r'"[^"]+"\s*:', source) is None
@@ -0,0 +1,48 @@
from __future__ import annotations
import httpx
import pytest
from app.crm import CrmClient, CrmOutcome
def make_client(handler) -> CrmClient:
client = CrmClient.__new__(CrmClient)
client._base_url = "https://portal.example/rest/1/token/"
client._host = "portal.example"
client._client = httpx.AsyncClient(
transport=httpx.MockTransport(handler), follow_redirects=False
)
return client
@pytest.mark.asyncio
async def test_crm_success_and_no_redirect() -> None:
client = make_client(
lambda request: httpx.Response(200, json={"result": {"ID": "42"}}, request=request)
)
result = await client.call("crm.contact.get", {"id": "42"}, mutating=False)
assert result.outcome == CrmOutcome.SUCCEEDED
assert result.result["ID"] == "42"
await client.close()
redirecting = make_client(
lambda request: httpx.Response(
302, headers={"Location": "https://evil.example/"}, request=request
)
)
result = await redirecting.call("crm.contact.get", {"id": "42"}, mutating=False)
assert result.outcome == CrmOutcome.PERMANENT
assert result.error_code == "crm_redirect_rejected"
await redirecting.close()
@pytest.mark.asyncio
async def test_mutating_timeout_is_uncertain() -> None:
def timeout(request):
raise httpx.ReadTimeout("timed out", request=request)
client = make_client(timeout)
result = await client.call("crm.contact.add", {"fields": {}}, mutating=True)
assert result.outcome == CrmOutcome.UNCERTAIN
await client.close()
@@ -0,0 +1,57 @@
from __future__ import annotations
import random
from datetime import UTC, datetime, timedelta
import pytest
from app.domain import (
ContactCandidate,
WorkflowState,
assert_transition,
choose_newest,
full_jitter_delay,
select_email,
validate_phone,
)
from app.mapping import CitizenshipDictionary, UnknownCitizenship
def test_contact_choice_has_numeric_id_tie_breaker() -> None:
created = datetime(2026, 8, 6, tzinfo=UTC)
selected = choose_newest(
[
ContactCandidate("9", created, None),
ContactCandidate("10", created, None),
]
)
assert selected and selected.b24_id == "10"
def test_phone_email_and_citizenship_mapping() -> None:
assert validate_phone("+79001234567") == "+79001234567"
with pytest.raises(ValueError):
validate_phone("8 900 123-45-67")
assert (
select_email(
[
{"VALUE": "home@example.test", "VALUE_TYPE": "HOME"},
{"VALUE": "work@example.test", "VALUE_TYPE": "WORK"},
]
)
== "work@example.test"
)
dictionary = CitizenshipDictionary(ttl_seconds=60)
now = datetime(2026, 8, 6, tzinfo=UTC)
dictionary.load([{"ID": "7", "VALUE": "Казахстан"}], now)
assert dictionary.resolve("7", now + timedelta(seconds=30)) == "Казахстан"
with pytest.raises(UnknownCitizenship):
dictionary.resolve("8", now)
def test_state_machine_and_retry_bounds() -> None:
assert_transition(WorkflowState.CREATED, WorkflowState.RUNNING)
with pytest.raises(ValueError):
assert_transition(WorkflowState.SUCCEEDED, WorkflowState.RUNNING)
delay = full_jitter_delay(4, 1, 5, rng=random.Random(1))
assert 0 <= delay <= 5
@@ -0,0 +1,72 @@
from __future__ import annotations
import uuid
from contextlib import asynccontextmanager
import pytest
from app.crm import CrmOutcome, CrmResult
from app.engine import WorkflowEngine
from app.repository import Profile
class Result:
rowcount = 1
class Connection:
async def execute(self, statement, params=None):
return Result()
class FakeRepository:
def __init__(self) -> None:
self.mapping = None
self.statements: list[str] = []
@asynccontextmanager
async def transaction(self):
yield Connection()
async def active_mapping(self, user_id):
return self.mapping
async def reserve_limiter_token(self, refill_per_second, burst):
return 0
class FakeCrm:
def __init__(self, user_id: uuid.UUID) -> None:
self.user_id = user_id
self.calls: list[tuple[str, dict]] = []
async def call(self, method, params, *, mutating):
self.calls.append((method, params))
if method == "crm.duplicate.findbycomm":
return CrmResult(CrmOutcome.SUCCEEDED, {"CONTACT": ["9", "10"]})
if method == "crm.contact.get":
contact_id = str(params["id"])
return CrmResult(
CrmOutcome.SUCCEEDED,
{
"ID": contact_id,
"CREATED_TIME": "2026-08-06T10:00:00Z",
"UF_CRM_100": None,
},
)
return CrmResult(CrmOutcome.SUCCEEDED, True)
@pytest.mark.asyncio
async def test_multiple_contacts_choose_numeric_newest(full_settings) -> None:
user_id = uuid.uuid4()
repository = FakeRepository()
crm = FakeCrm(user_id)
engine = WorkflowEngine(repository, crm, full_settings)
await engine._map_or_create(
uuid.uuid4(),
Profile(user_id=user_id, phone="+79001234567", identity_status="A", profile_status="A"),
)
updates = [params for method, params in crm.calls if method == "crm.contact.update"]
assert updates[0]["id"] == "10"
assert not any(method == "crm.contact.add" for method, _ in crm.calls)
@@ -0,0 +1,62 @@
from __future__ import annotations
import pytest
from app.security import (
WebhookValidationError,
parse_bounded_form,
redact,
token_matches,
validate_webhook,
)
def test_contact_webhook_contract(full_settings) -> None:
event = validate_webhook(
"contact",
{"token": "contact-token-value-32-characters", "ID": "42"},
{
"document_id[0]": "crm",
"document_id[1]": "CCrmDocumentContact",
"document_id[2]": "CONTACT_42",
"auth[domain]": "portal.example",
"auth[member_id]": "member_12345678",
"auth[client_endpoint]": "https://attacker.invalid/rest/",
},
"203.0.113.10",
full_settings,
alert_entity_type_id=None,
)
assert event.entity_id == "42"
def test_webhook_rejects_document_query_mismatch(full_settings) -> None:
with pytest.raises(WebhookValidationError, match="mismatch"):
validate_webhook(
"contact",
{"token": "contact-token-value-32-characters", "ID": "41"},
{
"document_id[0]": "crm",
"document_id[1]": "CCrmDocumentContact",
"document_id[2]": "CONTACT_42",
"auth[domain]": "portal.example",
"auth[member_id]": "member_12345678",
},
"203.0.113.10",
full_settings,
alert_entity_type_id=None,
)
def test_bounded_form_and_constant_time_token_helpers() -> None:
assert parse_bounded_form(b"a=1&b=2", max_fields=2) == {"a": "1", "b": "2"}
with pytest.raises(WebhookValidationError):
parse_bounded_form(b"a=1&b=2&c=3", max_fields=2)
assert token_matches("old", "new", "old")
assert not token_matches("other", "new", "old")
def test_redaction_removes_pii_and_secrets() -> None:
assert "secret-value" not in redact("token=secret-value")
assert redact("user@example.test") == "[PII_REDACTED]"
assert redact("+79001234567") == "[PII_REDACTED]"