Проект разделен на два репозитория

VM2_services/codebase/services/bitrix-sync/app/__init__.py

@@ -0,0 +1 @@
+"""HAN Bitrix24 synchronization service."""

VM2_services/codebase/services/bitrix-sync/app/config.py

@@ -0,0 +1,137 @@
+from __future__ import annotations
+
+import ipaddress
+import re
+from functools import cached_property
+from pathlib import Path
+from urllib.parse import urlsplit
+
+from pydantic import Field, SecretStr, model_validator
+from pydantic_settings import BaseSettings, SettingsConfigDict
+
+FIELD_RE = re.compile(r"^UF_CRM_[0-9]+$")
+MEMBER_RE = re.compile(r"^[A-Za-z0-9_-]{8,128}$")
+
+
+def _read_secret(value: SecretStr | None, path: Path | None) -> SecretStr | None:
+    if value and value.get_secret_value():
+        return value
+    if path:
+        text = path.read_text(encoding="utf-8").strip()
+        return SecretStr(text) if text else None
+    return None
+
+
+class Settings(BaseSettings):
+    model_config = SettingsConfigDict(env_prefix="BITRIX_SYNC_", extra="ignore")
+
+    enabled: bool = False
+    mode: str = "disabled"
+    database_url: SecretStr | None = None
+    database_url_file: Path | None = None
+    crm_rest_webhook_url: SecretStr | None = None
+    crm_rest_webhook_url_file: Path | None = None
+    contact_receiver_token: SecretStr | None = None
+    contact_receiver_token_file: Path | None = None
+    contact_receiver_previous_token: SecretStr | None = None
+    alert_receiver_token: SecretStr | None = None
+    alert_receiver_token_file: Path | None = None
+    alert_receiver_previous_token: SecretStr | None = None
+    service_token: SecretStr | None = None
+    service_token_file: Path | None = None
+
+    portal_host: str | None = None
+    portal_member_id: str | None = None
+    public_base_url: str | None = None
+    contact_user_id_field: str | None = None
+    contact_registered_field: str | None = None
+    contact_citizenship_field: str | None = None
+    webhook_allowed_cidrs: str = ""
+
+    http_timeout_sec: float = Field(default=10, ge=1, le=60)
+    db_pool_size: int = Field(default=5, ge=1, le=30)
+    webhook_max_body_bytes: int = Field(default=16_384, ge=1024, le=65_536)
+    webhook_max_fields: int = Field(default=24, ge=8, le=64)
+    lease_seconds: int = Field(default=60, ge=10, le=600)
+    claim_size: int = Field(default=20, ge=1, le=100)
+    batch_size: int = Field(default=20, ge=1, le=50)
+    batch_wait_ms: int = Field(default=200, ge=10, le=5000)
+    limiter_refill_per_sec: float = Field(default=2, gt=0, le=50)
+    limiter_burst: int = Field(default=2, ge=1, le=50)
+    max_in_flight: int = Field(default=2, ge=1, le=20)
+    retry_base_seconds: float = Field(default=1, ge=0.1, le=60)
+    retry_max_seconds: float = Field(default=900, ge=1, le=3600)
+    retry_horizon_seconds: int = Field(default=86_400, ge=60, le=604_800)
+    reconciliation_overlap_seconds: int = Field(default=300, ge=0, le=3600)
+    reconciliation_interval_seconds: int = Field(default=900, ge=60, le=86_400)
+
+    @model_validator(mode="after")
+    def validate_mode(self) -> Settings:
+        self.database_url = _read_secret(self.database_url, self.database_url_file)
+        self.crm_rest_webhook_url = _read_secret(
+            self.crm_rest_webhook_url, self.crm_rest_webhook_url_file
+        )
+        self.contact_receiver_token = _read_secret(
+            self.contact_receiver_token, self.contact_receiver_token_file
+        )
+        self.alert_receiver_token = _read_secret(
+            self.alert_receiver_token, self.alert_receiver_token_file
+        )
+        self.service_token = _read_secret(self.service_token, self.service_token_file)
+
+        expected_mode = "full" if self.enabled else "disabled"
+        if self.mode != expected_mode:
+            raise ValueError(f"mode must be {expected_mode!r} when enabled={self.enabled}")
+        if not self.enabled:
+            return self
+
+        required = {
+            "database_url": self.database_url,
+            "crm_rest_webhook_url": self.crm_rest_webhook_url,
+            "contact_receiver_token": self.contact_receiver_token,
+            "alert_receiver_token": self.alert_receiver_token,
+            "service_token": self.service_token,
+            "portal_host": self.portal_host,
+            "portal_member_id": self.portal_member_id,
+            "public_base_url": self.public_base_url,
+            "contact_user_id_field": self.contact_user_id_field,
+            "contact_registered_field": self.contact_registered_field,
+            "contact_citizenship_field": self.contact_citizenship_field,
+        }
+        missing = [name for name, value in required.items() if not value]
+        if missing:
+            raise ValueError("missing full-mode settings: " + ", ".join(missing))
+        for name in (
+            "contact_user_id_field",
+            "contact_registered_field",
+            "contact_citizenship_field",
+        ):
+            if not FIELD_RE.fullmatch(str(getattr(self, name))):
+                raise ValueError(f"{name} must match UF_CRM_<digits>")
+        if not MEMBER_RE.fullmatch(str(self.portal_member_id)):
+            raise ValueError("portal_member_id has invalid format")
+
+        crm = urlsplit(self.crm_rest_webhook_url.get_secret_value())
+        public = urlsplit(str(self.public_base_url))
+        if crm.scheme != "https" or crm.hostname != self.portal_host or crm.port not in (None, 443):
+            raise ValueError("CRM URL must be HTTPS on the approved portal host")
+        if public.scheme != "https" or not public.hostname or public.query or public.fragment:
+            raise ValueError("public_base_url must be a query-free HTTPS origin")
+        if not self.allowed_networks:
+            raise ValueError("webhook_allowed_cidrs cannot be empty in full mode")
+        return self
+
+    @cached_property
+    def allowed_networks(self) -> tuple[ipaddress.IPv4Network | ipaddress.IPv6Network, ...]:
+        values = [item.strip() for item in self.webhook_allowed_cidrs.split(",") if item.strip()]
+        return tuple(ipaddress.ip_network(item, strict=True) for item in values)
+
+    @staticmethod
+    def rest_field_name(field: str) -> str:
+        if not FIELD_RE.fullmatch(field):
+            raise ValueError("invalid Bitrix custom field")
+        return "ufCrm_" + field.removeprefix("UF_CRM_")
+
+
+def load_settings() -> Settings:
+    return Settings()

VM2_services/codebase/services/bitrix-sync/app/crm.py

@@ -0,0 +1,185 @@
+from __future__ import annotations
+
+import asyncio
+import ssl
+from dataclasses import dataclass
+from datetime import UTC, datetime
+from enum import StrEnum
+from typing import Any
+from urllib.parse import urljoin, urlsplit
+
+import httpx
+
+
+class CrmOutcome(StrEnum):
+    SUCCEEDED = "succeeded"
+    RETRY = "retry"
+    UNCERTAIN = "uncertain"
+    PERMANENT = "permanent"
+    RATE_LIMITED = "rate_limited"
+
+
+@dataclass(frozen=True)
+class CrmResult:
+    outcome: CrmOutcome
+    result: Any = None
+    error_code: str | None = None
+    http_status: int | None = None
+    retry_after: float | None = None
+
+
+class CrmClient:
+    """Host-locked, verified-TLS Bitrix client; redirects are never followed."""
+
+    def __init__(self, credential_url: str, approved_host: str, timeout: float) -> None:
+        parsed = urlsplit(credential_url)
+        if parsed.scheme != "https" or parsed.hostname != approved_host:
+            raise ValueError("credential URL is outside approved Bitrix host")
+        self._base_url = credential_url.rstrip("/") + "/"
+        self._host = approved_host
+        self._client = httpx.AsyncClient(
+            timeout=httpx.Timeout(timeout),
+            verify=ssl.create_default_context(),
+            follow_redirects=False,
+            limits=httpx.Limits(max_connections=4, max_keepalive_connections=2),
+            headers={"Accept": "application/json"},
+        )
+
+    async def close(self) -> None:
+        await self._client.aclose()
+
+    async def call(self, method: str, params: dict[str, Any], *, mutating: bool) -> CrmResult:
+        if method not in ALLOWED_METHODS:
+            raise ValueError("unapproved CRM method")
+        url = urljoin(self._base_url, method + ".json")
+        if urlsplit(url).hostname != self._host:
+            raise ValueError("CRM host changed during URL construction")
+        try:
+            response = await self._client.post(url, json=params)
+        except (httpx.ConnectError, httpx.ReadError, httpx.RemoteProtocolError):
+            return CrmResult(CrmOutcome.RETRY, error_code="crm_network")
+        except httpx.TimeoutException:
+            outcome = CrmOutcome.UNCERTAIN if mutating else CrmOutcome.RETRY
+            return CrmResult(outcome, error_code="crm_timeout")
+        if response.is_redirect:
+            return CrmResult(
+                CrmOutcome.PERMANENT,
+                error_code="crm_redirect_rejected",
+                http_status=response.status_code,
+            )
+        retry_after = _retry_after(response)
+        if response.status_code in (408, 429) or response.status_code >= 500:
+            return CrmResult(
+                CrmOutcome.RETRY,
+                error_code=f"crm_http_{response.status_code}",
+                http_status=response.status_code,
+                retry_after=retry_after,
+            )
+        try:
+            payload = response.json()
+        except ValueError:
+            return CrmResult(
+                CrmOutcome.PERMANENT,
+                error_code="crm_malformed_response",
+                http_status=response.status_code,
+            )
+        error = payload.get("error")
+        if error == "QUERY_LIMIT_EXCEEDED":
+            return CrmResult(CrmOutcome.RATE_LIMITED, error_code=error, retry_after=retry_after)
+        if error == "OPERATION_TIME_LIMIT":
+            return CrmResult(CrmOutcome.RETRY, error_code=error, retry_after=retry_after)
+        if error:
+            permanent = error in {
+                "ERROR_METHOD_NOT_FOUND",
+                "ERROR_WRONG_AUTH_TYPE",
+                "INVALID_CREDENTIALS",
+                "ACCESS_DENIED",
+                "ERROR_ARGUMENT",
+            }
+            return CrmResult(
+                CrmOutcome.PERMANENT if permanent else CrmOutcome.RETRY,
+                error_code=str(error)[:64],
+                http_status=response.status_code,
+            )
+        return CrmResult(CrmOutcome.SUCCEEDED, result=payload.get("result"))
+
+    async def batch(self, commands: list[tuple[str, dict[str, Any]]]) -> list[CrmResult]:
+        if not commands:
+            return []
+        if len(commands) > 50:
+            raise ValueError("Bitrix batch limit exceeded")
+        cmd = {
+            str(index): f"{method}?{httpx.QueryParams(params)}"
+            for index, (method, params) in enumerate(commands)
+            if method in ALLOWED_METHODS
+        }
+        batch = await self.call("batch", {"halt": 0, "cmd": cmd}, mutating=True)
+        if batch.outcome != CrmOutcome.SUCCEEDED:
+            return [batch for _ in commands]
+        result = batch.result or {}
+        successes = result.get("result", {})
+        errors = result.get("result_error", {})
+        return [
+            CrmResult(CrmOutcome.SUCCEEDED, result=successes.get(str(i)))
+            if str(i) in successes
+            else CrmResult(CrmOutcome.RETRY, error_code=str(errors.get(str(i), "batch_missing")))
+            for i in range(len(commands))
+        ]
+
+
+ALLOWED_METHODS = frozenset(
+    {
+        "batch",
+        "crm.duplicate.findbycomm",
+        "crm.contact.get",
+        "crm.contact.add",
+        "crm.contact.update",
+        "crm.contact.userfield.list",
+        "crm.item.list",
+        "crm.item.get",
+        "crm.item.add",
+        "crm.item.update",
+    }
+)
+
+
+def _retry_after(response: httpx.Response) -> float | None:
+    value = response.headers.get("Retry-After")
+    if not value:
+        return None
+    try:
+        return max(0.0, float(value))
+    except ValueError:
+        try:
+            retry_at = datetime.fromisoformat(value).astimezone(UTC)
+            return max(0.0, (retry_at - datetime.now(UTC)).total_seconds())
+        except ValueError:
+            return None
+
+
+class TokenBucket:
+    def __init__(self, refill_per_second: float, burst: int, max_in_flight: int) -> None:
+        self.refill_per_second = refill_per_second
+        self.burst = float(burst)
+        self.tokens = float(burst)
+        self.updated_at = asyncio.get_running_loop().time()
+        self._lock = asyncio.Lock()
+        self._slots = asyncio.Semaphore(max_in_flight)
+
+    async def acquire(self) -> None:
+        await self._slots.acquire()
+        while True:
+            async with self._lock:
+                now = asyncio.get_running_loop().time()
+                self.tokens = min(
+                    self.burst, self.tokens + (now - self.updated_at) * self.refill_per_second
+                )
+                self.updated_at = now
+                if self.tokens >= 1:
+                    self.tokens -= 1
+                    return
+                delay = (1 - self.tokens) / self.refill_per_second
+            await asyncio.sleep(delay)
+
+    def release(self) -> None:
+        self._slots.release()

VM2_services/codebase/services/bitrix-sync/app/domain.py

@@ -0,0 +1,106 @@
+from __future__ import annotations
+
+import hashlib
+import random
+import re
+from dataclasses import dataclass
+from datetime import UTC, datetime
+from enum import StrEnum
+from typing import Any
+
+PHONE_RE = re.compile(r"^\+7[0-9]{10}$")
+EMAIL_RE = re.compile(r"^[^@\s]+@[^@\s]+\.[^@\s]+$")
+
+
+class WorkflowState(StrEnum):
+    CREATED = "created"
+    RUNNING = "running"
+    WAITING_CRM = "waiting_crm"
+    WAITING_RETRY = "waiting_retry"
+    WAITING_MANUAL = "waiting_manual"
+    SUCCEEDED = "succeeded"
+    FAILED = "failed"
+    CANCELLED = "cancelled"
+
+
+TRANSITIONS: dict[WorkflowState, frozenset[WorkflowState]] = {
+    WorkflowState.CREATED: frozenset({WorkflowState.RUNNING, WorkflowState.CANCELLED}),
+    WorkflowState.RUNNING: frozenset(
+        {
+            WorkflowState.WAITING_CRM,
+            WorkflowState.WAITING_RETRY,
+            WorkflowState.WAITING_MANUAL,
+            WorkflowState.SUCCEEDED,
+            WorkflowState.FAILED,
+            WorkflowState.CANCELLED,
+        }
+    ),
+    WorkflowState.WAITING_CRM: frozenset(
+        {
+            WorkflowState.RUNNING,
+            WorkflowState.WAITING_RETRY,
+            WorkflowState.WAITING_MANUAL,
+            WorkflowState.FAILED,
+        }
+    ),
+    WorkflowState.WAITING_RETRY: frozenset(
+        {WorkflowState.RUNNING, WorkflowState.WAITING_MANUAL, WorkflowState.FAILED}
+    ),
+    WorkflowState.WAITING_MANUAL: frozenset(
+        {WorkflowState.RUNNING, WorkflowState.CANCELLED}
+    ),
+    WorkflowState.SUCCEEDED: frozenset(),
+    WorkflowState.FAILED: frozenset(),
+    WorkflowState.CANCELLED: frozenset(),
+}
+
+
+def assert_transition(current: WorkflowState, target: WorkflowState) -> None:
+    if target not in TRANSITIONS[current]:
+        raise ValueError(f"forbidden workflow transition {current} -> {target}")
+
+
+@dataclass(frozen=True)
+class ContactCandidate:
+    b24_id: str
+    created_at: datetime
+    crm_user_id: str | None
+
+
+def validate_phone(value: str) -> str:
+    if not PHONE_RE.fullmatch(value):
+        raise ValueError("phone must be Russian E.164 +7XXXXXXXXXX")
+    return value
+
+
+def choose_newest(candidates: list[ContactCandidate]) -> ContactCandidate | None:
+    if not candidates:
+        return None
+    return max(candidates, key=lambda item: (item.created_at, int(item.b24_id)))
+
+
+def select_email(items: list[dict[str, Any]]) -> str | None:
+    valid = [
+        item
+        for item in items
+        if isinstance(item.get("VALUE"), str) and EMAIL_RE.fullmatch(item["VALUE"])
+    ]
+    work = next((item for item in valid if item.get("VALUE_TYPE") == "WORK"), None)
+    selected = work or (valid[0] if valid else None)
+    return selected["VALUE"] if selected else None
+
+
+def safe_hash(value: str | None) -> str | None:
+    return hashlib.sha256(value.encode()).hexdigest() if value is not None else None
+
+
+def full_jitter_delay(
+    attempt: int, base: float, maximum: float, *, rng: random.Random | None = None
+) -> float:
+    ceiling = min(maximum, base * (2 ** max(0, attempt - 1)))
+    return (rng or random.SystemRandom()).uniform(0, ceiling)
+
+
+def parse_crm_datetime(value: str) -> datetime:
+    parsed = datetime.fromisoformat(value.replace("Z", "+00:00"))
+    return parsed.astimezone(UTC)

VM2_services/codebase/services/bitrix-sync/app/engine.py

@@ -0,0 +1,629 @@
+from __future__ import annotations
+
+import asyncio
+import uuid
+from dataclasses import dataclass, field
+from typing import Any
+
+from sqlalchemy import text
+
+from app.config import Settings
+from app.crm import CrmClient, CrmOutcome, CrmResult
+from app.domain import (
+    ContactCandidate,
+    choose_newest,
+    full_jitter_delay,
+    parse_crm_datetime,
+    select_email,
+    validate_phone,
+)
+from app.repository import LeasedTask, LeasedWebhook, Profile, Repository
+
+
+class BusinessConflict(Exception):
+    def __init__(self, code: str, candidates: list[str] | None = None) -> None:
+        self.code = code
+        self.candidates = candidates or []
+        super().__init__(code)
+
+
+@dataclass
+class WorkflowEngine:
+    repository: Repository
+    crm: CrmClient
+    settings: Settings
+    _in_flight: asyncio.Semaphore = field(init=False, repr=False)
+
+    def __post_init__(self) -> None:
+        self._in_flight = asyncio.Semaphore(self.settings.max_in_flight)
+
+    async def process(self, task: LeasedTask) -> None:
+        workflow_id = await self.repository.create_workflow(task)
+        profile = await self.repository.load_profile(task.user_id)
+        if profile is None:
+            await self._manual(workflow_id, "profile_missing", task.user_id)
+            await self.repository.complete_task(task, workflow_id)
+            return
+        try:
+            if task.task_type == "contact.map_or_create":
+                await self._map_or_create(workflow_id, profile)
+            elif task.task_type == "contact.update":
+                await self._update(workflow_id, profile)
+            elif task.task_type == "contact.deactivate":
+                await self._deactivate(workflow_id, profile)
+            else:
+                await self._technical_failure(workflow_id, "unknown_task_type")
+            await self.repository.complete_task(task, workflow_id)
+        except BusinessConflict as exc:
+            await self._alert(workflow_id, task.user_id, exc.code, exc.candidates)
+            await self._manual(workflow_id, exc.code, task.user_id)
+            await self.repository.complete_task(task, workflow_id)
+        except RetryableWorkflow as exc:
+            delay = exc.retry_after or full_jitter_delay(
+                task.attempt_count + 1,
+                self.settings.retry_base_seconds,
+                self.settings.retry_max_seconds,
+            )
+            await self.repository.retry_task(task, exc.code, delay)
+
+    async def process_webhook(self, item: LeasedWebhook) -> None:
+        if item.receiver_type == "alert":
+            await self.repository.complete_webhook(item)
+            return
+        user_id = await self.repository.mapped_user_for_external(item.external_id)
+        if user_id is None:
+            await self.repository.complete_webhook(item)
+            return
+        workflow_id = uuid.uuid4()
+        async with self.repository.transaction() as connection:
+            await connection.execute(
+                text(
+                    """
+                    INSERT INTO bitrix_sync.workflow_instances
+                      (id,workflow_type,user_id,external_id,state,current_step,deadline_at,
+                       created_at,updated_at)
+                    VALUES (:id,'contact.webhook',:user_id,:external_id,'running',
+                            'read_contact',now()+interval '24 hours',now(),now())
+                    """
+                ),
+                {"id": workflow_id, "user_id": user_id, "external_id": item.external_id},
+            )
+        try:
+            result = await self._command(
+                workflow_id,
+                "contact_get",
+                "crm.contact.get",
+                {"id": item.external_id, "select": self._select_fields()},
+                mutating=False,
+            )
+            contact = result.result
+            if str(contact.get(self.settings.contact_user_id_field)) != str(user_id):
+                await self._alert(
+                    workflow_id,
+                    user_id,
+                    "mapping_identity_mismatch",
+                    [item.external_id],
+                )
+                await self._manual(workflow_id, "mapping_identity_mismatch", user_id)
+                await self.repository.complete_webhook(item)
+                return
+            citizenship = await self._resolve_citizenship(
+                workflow_id, contact.get(self.settings.contact_citizenship_field)
+            )
+            source_value = contact.get("DATE_MODIFY") or contact.get("updatedTime")
+            await self.repository.apply_crm_profile(
+                user_id,
+                item.external_id,
+                full_name=contact.get("NAME") or None,
+                citizenship=citizenship,
+                email=select_email(contact.get("EMAIL") or []),
+                source_updated_at=parse_crm_datetime(source_value) if source_value else None,
+                source="reconciliation"
+                if item.event_type == "contact.reconciliation"
+                else "webhook",
+            )
+            await self.repository.complete_webhook(item)
+            async with self.repository.transaction() as connection:
+                await connection.execute(
+                    text(
+                        """
+                        UPDATE bitrix_sync.workflow_instances
+                        SET state='succeeded',current_step='done',
+                            completed_at=now(),updated_at=now()
+                        WHERE id=:id
+                        """
+                    ),
+                    {"id": workflow_id},
+                )
+        except RetryableWorkflow:
+            raise
+
+    async def _map_or_create(self, workflow_id: uuid.UUID, profile: Profile) -> None:
+        if profile.identity_status != "A" or profile.profile_status != "A":
+            await self._deactivate(workflow_id, profile)
+            return
+        validate_phone(profile.phone)
+        if mapped := await self.repository.active_mapping(profile.user_id):
+            await self._ensure_registered(workflow_id, mapped, profile.user_id)
+            return
+
+        found = await self._command(
+            workflow_id,
+            "duplicate_find",
+            "crm.duplicate.findbycomm",
+            {"type": "PHONE", "values": [profile.phone], "entity_type": "CONTACT"},
+            mutating=False,
+        )
+        ids = _contact_ids(found.result)
+        contacts: list[dict[str, Any]] = []
+        for contact_id in ids:
+            result = await self._command(
+                workflow_id,
+                "contact_get",
+                "crm.contact.get",
+                {"id": contact_id, "select": self._select_fields()},
+                mutating=False,
+            )
+            if isinstance(result.result, dict):
+                contacts.append(result.result)
+
+        same_user = next(
+            (
+                item
+                for item in contacts
+                if str(item.get(self.settings.contact_user_id_field)) == str(profile.user_id)
+            ),
+            None,
+        )
+        alert_code: str | None = None
+        if same_user:
+            selected_id = str(same_user["ID"])
+        elif not contacts:
+            selected_id = await self._create_contact(workflow_id, profile)
+        else:
+            candidates = [
+                ContactCandidate(
+                    str(item["ID"]),
+                    parse_crm_datetime(item.get("CREATED_TIME", "1970-01-01T00:00:00Z")),
+                    item.get(self.settings.contact_user_id_field),
+                )
+                for item in contacts
+            ]
+            selected = choose_newest(candidates)
+            assert selected is not None
+            all_ids = [item.b24_id for item in candidates]
+            if selected.crm_user_id and selected.crm_user_id != str(profile.user_id):
+                selected_id = await self._create_contact(workflow_id, profile)
+                alert_code = "contact_owned_by_other_user"
+            else:
+                selected_id = selected.b24_id
+                await self._write_identity(workflow_id, selected_id, profile.user_id, active=True)
+                if len(candidates) > 1:
+                    alert_code = "duplicate_contacts"
+            if alert_code:
+                await self._alert(workflow_id, profile.user_id, alert_code, all_ids)
+        await self._activate_mapping(workflow_id, profile.user_id, selected_id)
+
+    async def _create_contact(self, workflow_id: uuid.UUID, profile: Profile) -> str:
+        result = await self._command(
+            workflow_id,
+            "contact_add",
+            "crm.contact.add",
+            {
+                "fields": {
+                    "PHONE": [{"VALUE": profile.phone, "VALUE_TYPE": "WORK"}],
+                    self.settings.contact_user_id_field: str(profile.user_id),
+                    self.settings.contact_registered_field: "1",
+                }
+            },
+            mutating=True,
+        )
+        return str(result.result)
+
+    async def _update(self, workflow_id: uuid.UUID, profile: Profile) -> None:
+        validate_phone(profile.phone)
+        mapping = await self.repository.active_mapping(profile.user_id)
+        if mapping is None:
+            await self._coalesce_map_or_create(profile.user_id)
+            return
+        await self._command(
+            workflow_id,
+            "contact_update",
+            "crm.contact.update",
+            {
+                "id": mapping,
+                "fields": {"PHONE": [{"VALUE": profile.phone, "VALUE_TYPE": "WORK"}]},
+            },
+            mutating=True,
+        )
+
+    async def _deactivate(self, workflow_id: uuid.UUID, profile: Profile) -> None:
+        mapping = await self.repository.active_mapping(profile.user_id)
+        if mapping is None:
+            return
+        await self._write_identity(workflow_id, mapping, profile.user_id, active=False)
+        async with self.repository.transaction() as connection:
+            await connection.execute(
+                text(
+                    """
+                    UPDATE bitrix_sync.entity_external_mapping
+                    SET status='closed', closed_at=now(), close_reason='deactivated',
+                        workflow_id=:workflow_id, updated_at=now()
+                    WHERE entity_id=:user_id AND external_id=:external_id AND status='active'
+                    """
+                ),
+                {
+                    "workflow_id": workflow_id,
+                    "user_id": profile.user_id,
+                    "external_id": mapping,
+                },
+            )
+
+    async def process_rebind(self, request_id: uuid.UUID) -> None:
+        async with self.repository.transaction() as connection:
+            request = (
+                await connection.execute(
+                    text(
+                        """
+                        SELECT id,user_id,old_external_id,target_external_id,workflow_id
+                        FROM bitrix_sync.rebind_requests
+                        WHERE id=:id AND status IN ('pending','retry_wait')
+                        FOR UPDATE
+                        """
+                    ),
+                    {"id": request_id},
+                )
+            ).mappings().first()
+            if not request:
+                return
+            await connection.execute(
+                text(
+                    """
+                    UPDATE bitrix_sync.rebind_requests
+                    SET status='processing',updated_at=now() WHERE id=:id
+                    """
+                ),
+                {"id": request_id},
+            )
+        target = await self._command(
+            request["workflow_id"],
+            "rebind_target_get",
+            "crm.contact.get",
+            {"id": request["target_external_id"], "select": self._select_fields()},
+            mutating=False,
+        )
+        target_user = target.result.get(self.settings.contact_user_id_field)
+        if target_user and target_user != str(request["user_id"]):
+            raise BusinessConflict("rebind_target_owned", [request["target_external_id"]])
+        await self._write_identity(
+            request["workflow_id"], request["target_external_id"], request["user_id"], active=True
+        )
+        if request["old_external_id"]:
+            old = await self._command(
+                request["workflow_id"],
+                "rebind_old_get",
+                "crm.contact.get",
+                {"id": request["old_external_id"], "select": self._select_fields()},
+                mutating=False,
+            )
+            if old.result.get(self.settings.contact_user_id_field) == str(request["user_id"]):
+                await self._write_identity(
+                    request["workflow_id"],
+                    request["old_external_id"],
+                    request["user_id"],
+                    active=False,
+                )
+        async with self.repository.transaction() as connection:
+            await connection.execute(
+                text(
+                    """
+                    UPDATE bitrix_sync.entity_external_mapping
+                    SET status='closed',closed_at=now(),close_reason='rebind',
+                        workflow_id=:workflow_id,updated_at=now()
+                    WHERE entity_id=:user_id AND status='active'
+                    """
+                ),
+                {
+                    "workflow_id": request["workflow_id"],
+                    "user_id": request["user_id"],
+                },
+            )
+            await connection.execute(
+                text(
+                    """
+                    INSERT INTO bitrix_sync.entity_external_mapping
+                      (id,entity_type,entity_id,external_system,external_entity_type,
+                       external_id,status,opened_at,workflow_id,created_at,updated_at)
+                    VALUES (gen_random_uuid(),'contact',:user_id,'bitrix24','contact',
+                            :target,'active',now(),:workflow_id,now(),now())
+                    """
+                ),
+                {
+                    "workflow_id": request["workflow_id"],
+                    "user_id": request["user_id"],
+                    "target": request["target_external_id"],
+                },
+            )
+            await connection.execute(
+                text(
+                    """
+                    UPDATE bitrix_sync.rebind_requests
+                    SET status='succeeded',completed_at=now(),updated_at=now()
+                    WHERE id=:request_id
+                    """
+                ),
+                {
+                    "request_id": request_id,
+                },
+            )
+
+    async def _ensure_registered(
+        self, workflow_id: uuid.UUID, external_id: str, user_id: uuid.UUID
+    ) -> None:
+        contact = await self._command(
+            workflow_id,
+            "contact_get",
+            "crm.contact.get",
+            {"id": external_id, "select": self._select_fields()},
+            mutating=False,
+        )
+        if str(contact.result.get(self.settings.contact_registered_field)) not in {"1", "Y"}:
+            await self._write_identity(workflow_id, external_id, user_id, active=True)
+
+    async def _resolve_citizenship(
+        self, workflow_id: uuid.UUID, enum_id: str | int | None
+    ) -> str | None:
+        if enum_id in (None, ""):
+            return None
+        async with self.repository.transaction() as connection:
+            value = (
+                await connection.execute(
+                    text(
+                        """
+                        SELECT display_value FROM bitrix_sync.citizenship_dictionary
+                        WHERE enum_id=:enum_id AND expires_at>now()
+                        """
+                    ),
+                    {"enum_id": str(enum_id)},
+                )
+            ).scalar_one_or_none()
+        if value is not None:
+            return value
+        result = await self._command(
+            workflow_id,
+            "citizenship_fields_get",
+            "crm.contact.userfield.list",
+            {"filter": {"FIELD_NAME": self.settings.contact_citizenship_field}},
+            mutating=False,
+        )
+        fields = result.result if isinstance(result.result, list) else []
+        entries = fields[0].get("LIST", []) if fields else []
+        async with self.repository.transaction() as connection:
+            for entry in entries:
+                if entry.get("ID") is None or not isinstance(entry.get("VALUE"), str):
+                    continue
+                await connection.execute(
+                    text(
+                        """
+                        INSERT INTO bitrix_sync.citizenship_dictionary
+                          (enum_id,display_value,loaded_at,expires_at)
+                        VALUES (:id,:value,now(),now()+interval '1 hour')
+                        ON CONFLICT (enum_id) DO UPDATE
+                        SET display_value=excluded.display_value,loaded_at=now(),
+                            expires_at=excluded.expires_at
+                        """
+                    ),
+                    {"id": str(entry["ID"]), "value": entry["VALUE"]},
+                )
+        match = next(
+            (
+                entry["VALUE"]
+                for entry in entries
+                if str(entry.get("ID")) == str(enum_id)
+            ),
+            None,
+        )
+        if match is None:
+            raise BusinessConflict("unknown_citizenship_enum", [str(enum_id)])
+        return match
+
+    async def _write_identity(
+        self, workflow_id: uuid.UUID, external_id: str, user_id: uuid.UUID, *, active: bool
+    ) -> None:
+        fields: dict[str, Any] = {self.settings.contact_registered_field: "1" if active else "0"}
+        fields[self.settings.contact_user_id_field] = str(user_id) if active else ""
+        await self._command(
+            workflow_id,
+            "contact_update",
+            "crm.contact.update",
+            {"id": external_id, "fields": fields},
+            mutating=True,
+        )
+
+    async def _command(
+        self,
+        workflow_id: uuid.UUID,
+        command_type: str,
+        method: str,
+        params: dict[str, Any],
+        *,
+        mutating: bool,
+    ) -> CrmResult:
+        command_id = uuid.uuid4()
+        async with self.repository.transaction() as connection:
+            await connection.execute(
+                text(
+                    """
+                    INSERT INTO bitrix_sync.crm_commands
+                      (id,workflow_id,command_type,safe_request,status,attempt_count,
+                       next_attempt_at,created_at,updated_at)
+                    VALUES (:id,:workflow_id,:type,:safe_request,'in_flight',1,now(),now(),now())
+                    """
+                ),
+                {
+                    "id": command_id,
+                    "workflow_id": workflow_id,
+                    "type": command_type,
+                    "safe_request": {"keys": sorted(params), "method_class": method.split(".")[-1]},
+                },
+            )
+        while True:
+            limiter_delay = await self.repository.reserve_limiter_token(
+                self.settings.limiter_refill_per_sec, self.settings.limiter_burst
+            )
+            if limiter_delay <= 0:
+                break
+            await asyncio.sleep(limiter_delay)
+        async with self._in_flight:
+            result = await self.crm.call(method, params, mutating=mutating)
+        status = result.outcome.value
+        async with self.repository.transaction() as connection:
+            await connection.execute(
+                text(
+                    """
+                    UPDATE bitrix_sync.crm_commands
+                    SET status=:status,safe_error_code=:error,http_status=:http_status,
+                        safe_response=:safe_response,
+                        completed_at=CASE WHEN :terminal THEN now() END,
+                        updated_at=now()
+                    WHERE id=:id
+                    """
+                ),
+                {
+                    "id": command_id,
+                    "status": status,
+                    "error": result.error_code,
+                    "http_status": result.http_status,
+                    "safe_response": {"has_result": result.result is not None},
+                    "terminal": result.outcome in {CrmOutcome.SUCCEEDED, CrmOutcome.PERMANENT},
+                },
+            )
+        if result.outcome == CrmOutcome.SUCCEEDED:
+            return result
+        if result.outcome == CrmOutcome.PERMANENT:
+            await self._technical_failure(workflow_id, result.error_code or "crm_permanent")
+            raise BusinessConflict("technical_configuration_failure")
+        raise RetryableWorkflow(result.error_code or result.outcome.value, result.retry_after)
+
+    async def _activate_mapping(
+        self, workflow_id: uuid.UUID, user_id: uuid.UUID, external_id: str
+    ) -> None:
+        async with self.repository.transaction() as connection:
+            await connection.execute(
+                text(
+                    """
+                    INSERT INTO bitrix_sync.entity_external_mapping
+                      (id,entity_type,entity_id,external_system,external_entity_type,
+                       external_id,status,opened_at,workflow_id,created_at,updated_at)
+                    VALUES (gen_random_uuid(),'contact',:user_id,'bitrix24','contact',
+                            :external_id,'active',now(),:workflow_id,now(),now())
+                    ON CONFLICT DO NOTHING
+                    """
+                ),
+                {
+                    "workflow_id": workflow_id,
+                    "user_id": user_id,
+                    "external_id": external_id,
+                },
+            )
+
+    async def _coalesce_map_or_create(self, user_id: uuid.UUID) -> None:
+        async with self.repository.transaction() as connection:
+            await connection.execute(
+                text(
+                    """
+                    INSERT INTO han_app.sync_queue
+                      (id,task_type,entity_type,entity_id,dedup_key,payload_json,status,
+                       attempt_count,next_attempt_at,created_at,updated_at)
+                    VALUES (gen_random_uuid(),'contact.map_or_create','contact',:user_id,
+                            'contact.map_or_create:'||:user_id::text,
+                            jsonb_build_object('schema_version',1,'user_id',:user_id),
+                            'pending',0,now(),now(),now())
+                    ON CONFLICT DO NOTHING
+                    """
+                ),
+                {"user_id": user_id},
+            )
+
+    async def _alert(
+        self, workflow_id: uuid.UUID, user_id: uuid.UUID, alert_type: str, candidates: list[str]
+    ) -> None:
+        fingerprint = f"{alert_type}:{user_id}"
+        async with self.repository.transaction() as connection:
+            await connection.execute(
+                text(
+                    """
+                    INSERT INTO bitrix_sync.business_alerts
+                      (id,fingerprint,alert_type,severity,app_user_id,candidate_external_ids,
+                       workflow_id,status,occurrence_count,first_occurred_at,last_occurred_at,
+                       created_at,updated_at)
+                    VALUES (gen_random_uuid(),encode(digest(:fingerprint,'sha256'),'hex'),
+                            :type,'warning',:user_id,:candidates,:workflow_id,'open',1,
+                            now(),now(),now(),now())
+                    ON CONFLICT (alert_type,fingerprint) WHERE status='open'
+                    DO UPDATE SET occurrence_count=business_alerts.occurrence_count+1,
+                                  last_occurred_at=now(),updated_at=now()
+                    """
+                ),
+                {
+                    "fingerprint": fingerprint,
+                    "type": alert_type,
+                    "user_id": user_id,
+                    "candidates": candidates,
+                    "workflow_id": workflow_id,
+                },
+            )
+
+    async def _manual(self, workflow_id: uuid.UUID, code: str, user_id: uuid.UUID) -> None:
+        async with self.repository.transaction() as connection:
+            await connection.execute(
+                text(
+                    """
+                    UPDATE bitrix_sync.workflow_instances
+                    SET state='waiting_manual',outcome=:code,updated_at=now()
+                    WHERE id=:id
+                    """
+                ),
+                {"id": workflow_id, "code": code},
+            )
+
+    async def _technical_failure(self, workflow_id: uuid.UUID, code: str) -> None:
+        async with self.repository.transaction() as connection:
+            await connection.execute(
+                text(
+                    """
+                    INSERT INTO bitrix_sync.technical_dead_letters
+                      (id,workflow_id,operation,safe_error_code,failed_at,created_at)
+                    VALUES (gen_random_uuid(),:workflow_id,'crm_command',:code,now(),now())
+                    """
+                ),
+                {"workflow_id": workflow_id, "code": code[:64]},
+            )
+
+    def _select_fields(self) -> list[str]:
+        return [
+            "ID",
+            "NAME",
+            "PHONE",
+            "EMAIL",
+            "CREATED_TIME",
+            "DATE_MODIFY",
+            self.settings.contact_user_id_field,
+            self.settings.contact_registered_field,
+            self.settings.contact_citizenship_field,
+        ]
+
+
+class RetryableWorkflow(Exception):
+    def __init__(self, code: str, retry_after: float | None = None) -> None:
+        self.code = code
+        self.retry_after = retry_after
+        super().__init__(code)
+
+
+def _contact_ids(result: Any) -> list[str]:
+    if isinstance(result, dict):
+        values = result.get("CONTACT", [])
+    else:
+        values = result or []
+    return sorted({str(value) for value in values if str(value).isdigit()}, key=int)

VM2_services/codebase/services/bitrix-sync/app/main.py

@@ -0,0 +1,173 @@
+from __future__ import annotations
+
+import hmac
+from contextlib import asynccontextmanager
+from typing import Annotated
+
+import uvicorn
+from fastapi import Depends, FastAPI, Header, HTTPException, Query, Request, Response
+from fastapi.responses import JSONResponse
+from sqlalchemy import text
+
+from app.config import Settings, load_settings
+from app.repository import Repository
+from app.security import WebhookValidationError, parse_bounded_form, validate_webhook
+
+
+@asynccontextmanager
+async def lifespan(app: FastAPI):
+    settings = load_settings()
+    app.state.settings = settings
+    app.state.repository = (
+        Repository(settings.database_url.get_secret_value(), settings.db_pool_size)
+        if settings.enabled and settings.database_url
+        else None
+    )
+    yield
+    if app.state.repository:
+        await app.state.repository.close()
+
+
+app = FastAPI(
+    title="HAN Bitrix Sync",
+    version="0.1.0",
+    docs_url=None,
+    redoc_url=None,
+    lifespan=lifespan,
+)
+
+
+def settings(request: Request) -> Settings:
+    return request.app.state.settings
+
+
+def repository(request: Request) -> Repository:
+    repo = request.app.state.repository
+    if repo is None:
+        raise HTTPException(status_code=503, detail="sync_disabled")
+    return repo
+
+
+async def require_service_token(
+    request: Request,
+    authorization: Annotated[str | None, Header()] = None,
+) -> None:
+    configured = settings(request).service_token
+    expected = f"Bearer {configured.get_secret_value()}" if configured else ""
+    if not authorization or not hmac.compare_digest(authorization, expected):
+        raise HTTPException(status_code=401, detail="unauthorized")
+
+
+@app.get("/health/live", include_in_schema=True)
+async def live() -> dict[str, str]:
+    return {"status": "live"}
+
+
+@app.get("/health/ready", include_in_schema=True)
+async def ready(request: Request) -> Response:
+    config = settings(request)
+    if not config.enabled:
+        return JSONResponse(
+            status_code=503,
+            content={"status": "not_ready", "reason": "sync_disabled"},
+        )
+    repo = repository(request)
+    if not await repo.ping():
+        return JSONResponse(status_code=503, content={"status": "not_ready", "reason": "database"})
+    return JSONResponse({"status": "ready", "mode": config.mode})
+
+
+@app.get(
+    "/internal/sync/v1/status",
+    dependencies=[Depends(require_service_token)],
+    include_in_schema=True,
+)
+async def sync_status(request: Request) -> dict:
+    result = await repository(request).status()
+    result["mode"] = settings(request).mode
+    return result
+
+
+@app.post("/bitrix/sync/webhook/contact", status_code=202, include_in_schema=True)
+async def contact_webhook(
+    request: Request,
+    token: Annotated[str | None, Query(max_length=256)] = None,
+    ID: Annotated[str | None, Query(pattern=r"^[1-9][0-9]{0,19}$")] = None,  # noqa: N803
+) -> Response:
+    return await _receive(request, "contact", {"token": token or "", "ID": ID or ""})
+
+
+@app.post("/bitrix/sync/webhook/alert", status_code=202, include_in_schema=True)
+async def alert_webhook(
+    request: Request,
+    token: Annotated[str | None, Query(max_length=256)] = None,
+    ID: Annotated[str | None, Query(pattern=r"^[1-9][0-9]{0,19}$")] = None,  # noqa: N803
+) -> Response:
+    return await _receive(request, "alert", {"token": token or "", "ID": ID or ""})
+
+
+async def _receive(request: Request, receiver: str, query: dict[str, str]) -> Response:
+    config = settings(request)
+    if not config.enabled:
+        raise HTTPException(status_code=503, detail="sync_disabled")
+    if request.headers.get("content-type", "").split(";", 1)[0].lower() != (
+        "application/x-www-form-urlencoded"
+    ):
+        raise HTTPException(status_code=400, detail="invalid_content_type")
+    content_length = request.headers.get("content-length")
+    if content_length and (
+        not content_length.isdigit() or int(content_length) > config.webhook_max_body_bytes
+    ):
+        raise HTTPException(status_code=413, detail="body_too_large")
+    body = await request.body()
+    if len(body) > config.webhook_max_body_bytes:
+        raise HTTPException(status_code=413, detail="body_too_large")
+    form = parse_bounded_form(body, max_fields=config.webhook_max_fields)
+    # The container is reachable only from the trusted VM2 nginx network.
+    # nginx overwrites X-Real-IP from the TCP peer after its CIDR check.
+    source_ip = request.headers.get("x-real-ip") or (request.client.host if request.client else "")
+    alert_entity_type_id = (
+        await _alert_entity_type(repository(request)) if receiver == "alert" else None
+    )
+    try:
+        event = validate_webhook(
+            receiver,
+            query,
+            form,
+            source_ip,
+            config,
+            alert_entity_type_id=alert_entity_type_id,
+        )
+    except PermissionError as exc:
+        raise HTTPException(status_code=403, detail="forbidden") from exc
+    except WebhookValidationError as exc:
+        raise HTTPException(status_code=400, detail="malformed_webhook") from exc
+    await repository(request).insert_webhook(
+        event.receiver_type, event.event_type, event.entity_id, event.source_ip
+    )
+    return Response(status_code=202)
+
+
+async def _alert_entity_type(repo: Repository) -> int | None:
+    async with repo.engine.connect() as connection:
+        value = (
+            await connection.execute(
+                text(
+                    """
+                    SELECT (value_json->>'entity_type_id')::integer
+                    FROM bitrix_sync.settings
+                    WHERE key='business_alerts' AND active=true AND validation_status='valid'
+                    """
+                )
+            )
+        ).scalar_one_or_none()
+        return value
+
+
+def run() -> None:
+    uvicorn.run(
+        "app.main:app",
+        host="0.0.0.0",  # noqa: S104 - container-only port, not host-published
+        port=8080,
+        proxy_headers=False,
+    )

VM2_services/codebase/services/bitrix-sync/app/mapping.py

@@ -0,0 +1,56 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+from datetime import UTC, datetime, timedelta
+from typing import Any
+
+from app.domain import select_email
+
+
+class UnknownCitizenship(ValueError):
+    pass
+
+
+@dataclass(frozen=True)
+class CitizenshipEntry:
+    enum_id: str
+    display_value: str
+    loaded_at: datetime
+
+
+class CitizenshipDictionary:
+    def __init__(self, ttl_seconds: int = 3600) -> None:
+        self.ttl = timedelta(seconds=ttl_seconds)
+        self._entries: dict[str, CitizenshipEntry] = {}
+        self.loaded_at: datetime | None = None
+
+    def load(self, values: list[dict[str, Any]], now: datetime | None = None) -> None:
+        loaded_at = now or datetime.now(UTC)
+        self._entries = {
+            str(item["ID"]): CitizenshipEntry(
+                str(item["ID"]), str(item["VALUE"]), loaded_at
+            )
+            for item in values
+            if item.get("ID") is not None and isinstance(item.get("VALUE"), str)
+        }
+        self.loaded_at = loaded_at
+
+    def resolve(self, enum_id: str | int | None, now: datetime | None = None) -> str | None:
+        if enum_id in (None, ""):
+            return None
+        entry = self._entries.get(str(enum_id))
+        if entry is None:
+            raise UnknownCitizenship(str(enum_id))
+        if (now or datetime.now(UTC)) - entry.loaded_at > self.ttl:
+            raise UnknownCitizenship(str(enum_id))
+        return entry.display_value
+
+
+def crm_master_projection(
+    contact: dict[str, Any], citizenship: CitizenshipDictionary, citizenship_field: str
+) -> dict[str, str | None]:
+    return {
+        "full_name": contact.get("NAME") or None,
+        "email": select_email(contact.get("EMAIL") or []),
+        "citizenship": citizenship.resolve(contact.get(citizenship_field)),
+    }

VM2_services/codebase/services/bitrix-sync/app/reconciliation.py

@@ -0,0 +1,139 @@
+from __future__ import annotations
+
+import asyncio
+from datetime import UTC, datetime, timedelta
+from typing import Any
+
+from sqlalchemy import text
+
+from app.config import load_settings
+from app.crm import CrmClient, CrmOutcome
+from app.repository import Repository
+
+
+class IncrementalReconciler:
+    def __init__(self, repository: Repository, crm: CrmClient, registered_rest_field: str) -> None:
+        self.repository = repository
+        self.crm = crm
+        self.registered_rest_field = registered_rest_field
+
+    async def run_once(self, overlap_seconds: int) -> int:
+        async with self.repository.transaction() as connection:
+            acquired = (
+                await connection.execute(
+                    text(
+                        """
+                        SELECT pg_try_advisory_xact_lock(
+                          hashtext('bitrix-contact-reconciliation')
+                        )
+                        """
+                    )
+                )
+            ).scalar_one()
+            if not acquired:
+                return 0
+            cursor = (
+                await connection.execute(
+                    text(
+                        """
+                        SELECT watermark FROM bitrix_sync.reconciliation_cursors
+                        WHERE job_type='contact_incremental' FOR UPDATE
+                        """
+                    )
+                )
+            ).scalar_one_or_none()
+        started_at = datetime.now(UTC)
+        since = (cursor or datetime(1970, 1, 1, tzinfo=UTC)) - timedelta(seconds=overlap_seconds)
+        start = 0
+        scanned: list[str] = []
+        while True:
+            result = await self.crm.call(
+                "crm.item.list",
+                {
+                    "entityTypeId": 3,
+                    "select": ["id"],
+                    "filter": {
+                        ">=updatedTime": since.replace(tzinfo=None).isoformat(timespec="seconds"),
+                        "opened": 1,
+                        self.registered_rest_field: 1,
+                    },
+                    "start": start,
+                },
+                mutating=False,
+            )
+            if result.outcome != CrmOutcome.SUCCEEDED:
+                raise RuntimeError(result.error_code or "reconciliation_failed")
+            payload: dict[str, Any] = result.result or {}
+            items = payload.get("items", payload if isinstance(payload, list) else [])
+            scanned.extend(str(item["id"]) for item in items if "id" in item)
+            next_start = payload.get("next")
+            if next_start is None:
+                break
+            start = int(next_start)
+        await self._enqueue_changed(scanned)
+        async with self.repository.transaction() as connection:
+            await connection.execute(
+                text(
+                    """
+                    INSERT INTO bitrix_sync.reconciliation_cursors
+                      (job_type,watermark,overlap_seconds,last_success_at,last_scanned_count,
+                       created_at,updated_at)
+                    VALUES ('contact_incremental',:watermark,:overlap,now(),:count,now(),now())
+                    ON CONFLICT (job_type) DO UPDATE
+                    SET watermark=excluded.watermark,overlap_seconds=excluded.overlap_seconds,
+                        last_success_at=now(),last_scanned_count=excluded.last_scanned_count,
+                        updated_at=now()
+                    """
+                ),
+                {"watermark": started_at, "overlap": overlap_seconds, "count": len(scanned)},
+            )
+        return len(scanned)
+
+    async def _enqueue_changed(self, external_ids: list[str]) -> None:
+        if not external_ids:
+            return
+        async with self.repository.transaction() as connection:
+            await connection.execute(
+                text(
+                    """
+                    INSERT INTO bitrix_sync.webhook_inbox
+                      (id,receiver_type,event_type,external_entity_id,status,coalesced_count,
+                       received_at,last_received_at)
+                    SELECT gen_random_uuid(),'contact','contact.reconciliation',id,'received',1,
+                           now(),now()
+                    FROM unnest(CAST(:ids AS text[])) id
+                    WHERE NOT EXISTS (
+                      SELECT 1 FROM bitrix_sync.webhook_inbox w
+                      WHERE w.receiver_type='contact' AND w.external_entity_id=id
+                        AND w.status IN ('received','processing','retry_wait')
+                    )
+                    """
+                ),
+                {"ids": external_ids},
+            )
+
+
+async def reconciliation_main() -> None:
+    settings = load_settings()
+    if not settings.enabled:
+        return
+    assert settings.database_url and settings.crm_rest_webhook_url and settings.portal_host
+    assert settings.contact_registered_field
+    repository = Repository(settings.database_url.get_secret_value(), settings.db_pool_size)
+    crm = CrmClient(
+        settings.crm_rest_webhook_url.get_secret_value(),
+        settings.portal_host,
+        settings.http_timeout_sec,
+    )
+    reconciler = IncrementalReconciler(
+        repository, crm, settings.rest_field_name(settings.contact_registered_field)
+    )
+    try:
+        await reconciler.run_once(settings.reconciliation_overlap_seconds)
+    finally:
+        await crm.close()
+        await repository.close()
+
+
+def run() -> None:
+    asyncio.run(reconciliation_main())

VM2_services/codebase/services/bitrix-sync/app/repository.py

@@ -0,0 +1,533 @@
+from __future__ import annotations
+
+import os
+import ssl
+import uuid
+from collections.abc import AsyncIterator
+from contextlib import asynccontextmanager
+from dataclasses import dataclass
+from datetime import UTC, datetime
+from typing import Any
+
+from sqlalchemy import text
+from sqlalchemy.ext.asyncio import AsyncConnection, AsyncEngine, create_async_engine
+
+
+def postgres_ssl_context() -> ssl.SSLContext:
+    ca_file = os.environ.get("PG_CA_FILE")
+    if not ca_file:
+        raise RuntimeError("PG_CA_FILE is required")
+    context = ssl.create_default_context(cafile=ca_file)
+    context.check_hostname = True
+    context.verify_mode = ssl.CERT_REQUIRED
+    return context
+
+
+@dataclass(frozen=True)
+class LeasedTask:
+    id: uuid.UUID
+    task_type: str
+    user_id: uuid.UUID
+    lease_token: uuid.UUID
+    attempt_count: int
+
+
+@dataclass(frozen=True)
+class LeasedWebhook:
+    id: uuid.UUID
+    receiver_type: str
+    event_type: str
+    external_id: str
+    lease_token: uuid.UUID
+    attempt_count: int
+
+
+@dataclass(frozen=True)
+class Profile:
+    user_id: uuid.UUID
+    phone: str
+    identity_status: str
+    profile_status: str
+
+
+class Repository:
+    def __init__(self, database_url: str, pool_size: int = 5) -> None:
+        self.engine: AsyncEngine = create_async_engine(
+            database_url,
+            pool_size=pool_size,
+            pool_pre_ping=True,
+            connect_args={"ssl": postgres_ssl_context()},
+        )
+
+    async def close(self) -> None:
+        await self.engine.dispose()
+
+    @asynccontextmanager
+    async def transaction(self) -> AsyncIterator[AsyncConnection]:
+        async with self.engine.begin() as connection:
+            yield connection
+
+    async def ping(self) -> bool:
+        try:
+            async with self.engine.connect() as connection:
+                await connection.execute(text("SELECT 1"))
+            return True
+        except Exception:
+            return False
+
+    async def claim_tasks(
+        self, worker_id: str, limit: int, lease_seconds: int
+    ) -> list[LeasedTask]:
+        sql = text(
+            """
+            WITH candidates AS (
+              SELECT id
+              FROM han_app.sync_queue
+              WHERE status IN ('pending','retry_wait')
+                AND next_attempt_at <= now()
+                AND (locked_until IS NULL OR locked_until < now())
+              ORDER BY next_attempt_at, created_at
+              FOR UPDATE SKIP LOCKED
+              LIMIT :limit
+            )
+            UPDATE han_app.sync_queue q
+            SET status='leased', locked_by=:worker_id,
+                locked_until=now() + make_interval(secs => :lease_seconds),
+                lease_token=gen_random_uuid(), updated_at=now()
+            FROM candidates c
+            WHERE q.id=c.id
+            RETURNING q.id, q.task_type, q.entity_id, q.lease_token, q.attempt_count
+            """
+        )
+        async with self.engine.begin() as connection:
+            rows = (
+                await connection.execute(
+                    sql,
+                    {"worker_id": worker_id, "limit": limit, "lease_seconds": lease_seconds},
+                )
+            ).mappings()
+            return [
+                LeasedTask(
+                    row["id"],
+                    row["task_type"],
+                    row["entity_id"],
+                    row["lease_token"],
+                    row["attempt_count"],
+                )
+                for row in rows
+            ]
+
+    async def load_profile(self, user_id: uuid.UUID) -> Profile | None:
+        sql = text(
+            """
+            SELECT i.id user_id, i.phone_number phone, i.record_status identity_status,
+                   p.record_status profile_status
+            FROM han_app.user_identities i
+            JOIN han_app.client_profiles p ON p.user_id=i.id
+            WHERE i.id=:user_id
+            """
+        )
+        async with self.engine.connect() as connection:
+            row = (await connection.execute(sql, {"user_id": user_id})).mappings().first()
+            return Profile(**row) if row else None
+
+    async def active_mapping(self, user_id: uuid.UUID) -> str | None:
+        sql = text(
+            """
+            SELECT external_id FROM bitrix_sync.entity_external_mapping
+            WHERE external_system='bitrix24' AND entity_type='contact'
+              AND entity_id=:user_id AND status='active'
+            """
+        )
+        async with self.engine.connect() as connection:
+            return (await connection.execute(sql, {"user_id": user_id})).scalar_one_or_none()
+
+    async def create_workflow(self, task: LeasedTask) -> uuid.UUID:
+        workflow_id = uuid.uuid4()
+        sql = text(
+            """
+            INSERT INTO bitrix_sync.workflow_instances
+              (id, workflow_type, user_id, state, current_step, source_task_id,
+               deadline_at, created_at, updated_at)
+            VALUES (:id, :workflow_type, :user_id, 'created', 'load_profile', :task_id,
+                    now() + interval '24 hours', now(), now())
+            ON CONFLICT (source_task_id) DO UPDATE SET updated_at=now()
+            RETURNING id
+            """
+        )
+        async with self.engine.begin() as connection:
+            return (
+                await connection.execute(
+                    sql,
+                    {
+                        "id": workflow_id,
+                        "workflow_type": task.task_type,
+                        "user_id": task.user_id,
+                        "task_id": task.id,
+                    },
+                )
+            ).scalar_one()
+
+    async def complete_task(self, task: LeasedTask, workflow_id: uuid.UUID) -> bool:
+        async with self.engine.begin() as connection:
+            result = await connection.execute(
+                text(
+                    """
+                    UPDATE han_app.sync_queue
+                    SET status='processed', completed_at=now(), locked_by=NULL,
+                        locked_until=NULL, lease_token=NULL, updated_at=now()
+                    WHERE id=:id AND status='leased' AND lease_token=:lease_token
+                    """
+                ),
+                {"id": task.id, "lease_token": task.lease_token},
+            )
+            if result.rowcount != 1:
+                return False
+            await connection.execute(
+                text(
+                    """
+                    UPDATE bitrix_sync.workflow_instances
+                    SET state='succeeded', current_step='done', outcome='processed',
+                        completed_at=now(), updated_at=now()
+                    WHERE id=:workflow_id AND state NOT IN ('succeeded','failed','cancelled')
+                    """
+                ),
+                {"workflow_id": workflow_id},
+            )
+            return True
+
+    async def retry_task(
+        self, task: LeasedTask, safe_code: str, delay_seconds: float
+    ) -> bool:
+        sql = text(
+            """
+            UPDATE han_app.sync_queue
+            SET status='retry_wait', attempt_count=attempt_count+1,
+                next_attempt_at=now() + make_interval(secs => :delay),
+                last_error_code=:code, last_error_at=now(),
+                locked_by=NULL, locked_until=NULL, lease_token=NULL, updated_at=now()
+            WHERE id=:id AND status='leased' AND lease_token=:lease_token
+            """
+        )
+        async with self.engine.begin() as connection:
+            result = await connection.execute(
+                sql,
+                {
+                    "id": task.id,
+                    "lease_token": task.lease_token,
+                    "code": safe_code[:64],
+                    "delay": delay_seconds,
+                },
+            )
+            return result.rowcount == 1
+
+    async def insert_webhook(
+        self,
+        receiver_type: str,
+        event_type: str,
+        entity_id: str,
+        source_ip: str,
+    ) -> uuid.UUID:
+        inbox_id = uuid.uuid4()
+        async with self.engine.begin() as connection:
+            existing = await connection.execute(
+                text(
+                    """
+                    SELECT id FROM bitrix_sync.webhook_inbox
+                    WHERE receiver_type=:receiver AND external_entity_id=:entity_id
+                      AND status IN ('received','processing','retry_wait')
+                    FOR UPDATE
+                    """
+                ),
+                {"receiver": receiver_type, "entity_id": entity_id},
+            )
+            if row := existing.first():
+                await connection.execute(
+                    text(
+                        """
+                        UPDATE bitrix_sync.webhook_inbox
+                        SET coalesced_count=coalesced_count+1, last_received_at=now()
+                        WHERE id=:id
+                        """
+                    ),
+                    {"id": row[0]},
+                )
+                return row[0]
+            await connection.execute(
+                text(
+                    """
+                    INSERT INTO bitrix_sync.webhook_inbox
+                      (id, receiver_type, event_type, external_entity_id, source_ip,
+                       status, coalesced_count, received_at, last_received_at)
+                    VALUES (:id,:receiver,:event,:entity_id,CAST(:source_ip AS inet),
+                            'received',1,now(),now())
+                    """
+                ),
+                {
+                    "id": inbox_id,
+                    "receiver": receiver_type,
+                    "event": event_type,
+                    "entity_id": entity_id,
+                    "source_ip": source_ip,
+                },
+            )
+        return inbox_id
+
+    async def claim_webhooks(
+        self, worker_id: str, limit: int, lease_seconds: int
+    ) -> list[LeasedWebhook]:
+        sql = text(
+            """
+            WITH candidates AS (
+              SELECT id FROM bitrix_sync.webhook_inbox
+              WHERE status IN ('received','retry_wait') AND next_attempt_at<=now()
+                AND (locked_until IS NULL OR locked_until<now())
+              ORDER BY next_attempt_at,received_at
+              FOR UPDATE SKIP LOCKED LIMIT :limit
+            )
+            UPDATE bitrix_sync.webhook_inbox w
+            SET status='processing',locked_by=:worker_id,
+                locked_until=now()+make_interval(secs=>:lease_seconds),
+                lease_token=gen_random_uuid()
+            FROM candidates c WHERE w.id=c.id
+            RETURNING w.id,w.receiver_type,w.event_type,w.external_entity_id,
+                      w.lease_token,w.attempt_count
+            """
+        )
+        async with self.engine.begin() as connection:
+            rows = (
+                await connection.execute(
+                    sql,
+                    {"worker_id": worker_id, "limit": limit, "lease_seconds": lease_seconds},
+                )
+            ).mappings()
+            return [
+                LeasedWebhook(
+                    row["id"],
+                    row["receiver_type"],
+                    row["event_type"],
+                    row["external_entity_id"],
+                    row["lease_token"],
+                    row["attempt_count"],
+                )
+                for row in rows
+            ]
+
+    async def mapped_user_for_external(self, external_id: str) -> uuid.UUID | None:
+        async with self.engine.connect() as connection:
+            return (
+                await connection.execute(
+                    text(
+                        """
+                        SELECT entity_id FROM bitrix_sync.entity_external_mapping
+                        WHERE external_system='bitrix24' AND external_entity_type='contact'
+                          AND external_id=:external_id AND status='active'
+                        """
+                    ),
+                    {"external_id": external_id},
+                )
+            ).scalar_one_or_none()
+
+    async def apply_crm_profile(
+        self,
+        user_id: uuid.UUID,
+        external_id: str,
+        *,
+        full_name: str | None,
+        citizenship: str | None,
+        email: str | None,
+        source_updated_at: datetime | None,
+        source: str,
+    ) -> None:
+        async with self.engine.begin() as connection:
+            await connection.execute(text("SET LOCAL han.sync_suppress='true'"))
+            await connection.execute(
+                text(
+                    """
+                    UPDATE han_app.client_profiles
+                    SET full_name=:full_name,citizenship=:citizenship,email=:email,updated_at=now()
+                    WHERE user_id=:user_id AND record_status='A'
+                    """
+                ),
+                {
+                    "user_id": user_id,
+                    "full_name": full_name,
+                    "citizenship": citizenship,
+                    "email": email,
+                },
+            )
+            await connection.execute(
+                text(
+                    """
+                    INSERT INTO bitrix_sync.contact_snapshots
+                      (id,mapping_id,user_id,external_id,full_name_hash,email_hash,
+                       citizenship_hash,source_updated_at,last_applied_source,
+                       last_webhook_received_at,created_at,updated_at)
+                    SELECT gen_random_uuid(),m.id,:user_id,:external_id,
+                           encode(digest(coalesce(:full_name,''),'sha256'),'hex'),
+                           encode(digest(coalesce(:email,''),'sha256'),'hex'),
+                           encode(digest(coalesce(:citizenship,''),'sha256'),'hex'),
+                           :source_updated_at,:source,
+                           CASE WHEN :source='webhook' THEN now() END,now(),now()
+                    FROM bitrix_sync.entity_external_mapping m
+                    WHERE m.entity_id=:user_id AND m.external_id=:external_id AND m.status='active'
+                    ON CONFLICT (mapping_id) DO UPDATE
+                    SET full_name_hash=excluded.full_name_hash,email_hash=excluded.email_hash,
+                        citizenship_hash=excluded.citizenship_hash,
+                        source_updated_at=excluded.source_updated_at,
+                        last_applied_source=excluded.last_applied_source,
+                        last_webhook_received_at=coalesce(
+                          excluded.last_webhook_received_at,
+                          bitrix_sync.contact_snapshots.last_webhook_received_at),
+                        updated_at=now()
+                    """
+                ),
+                {
+                    "user_id": user_id,
+                    "external_id": external_id,
+                    "full_name": full_name,
+                    "citizenship": citizenship,
+                    "email": email,
+                    "source_updated_at": source_updated_at,
+                    "source": source,
+                },
+            )
+
+    async def complete_webhook(self, item: LeasedWebhook) -> bool:
+        async with self.engine.begin() as connection:
+            result = await connection.execute(
+                text(
+                    """
+                    UPDATE bitrix_sync.webhook_inbox
+                    SET status='processed',processed_at=now(),locked_by=NULL,
+                        locked_until=NULL,lease_token=NULL
+                    WHERE id=:id AND status='processing' AND lease_token=:lease_token
+                    """
+                ),
+                {"id": item.id, "lease_token": item.lease_token},
+            )
+            return result.rowcount == 1
+
+    async def retry_webhook(
+        self, item: LeasedWebhook, safe_code: str, delay_seconds: float
+    ) -> bool:
+        async with self.engine.begin() as connection:
+            result = await connection.execute(
+                text(
+                    """
+                    UPDATE bitrix_sync.webhook_inbox
+                    SET status='retry_wait',attempt_count=attempt_count+1,
+                        next_attempt_at=now()+make_interval(secs=>:delay),
+                        safe_error_code=:code,locked_by=NULL,locked_until=NULL,lease_token=NULL
+                    WHERE id=:id AND status='processing' AND lease_token=:lease_token
+                    """
+                ),
+                {
+                    "id": item.id,
+                    "lease_token": item.lease_token,
+                    "code": safe_code[:64],
+                    "delay": delay_seconds,
+                },
+            )
+            return result.rowcount == 1
+
+    async def pending_rebind_ids(self, limit: int) -> list[uuid.UUID]:
+        async with self.engine.connect() as connection:
+            rows = await connection.execute(
+                text(
+                    """
+                    SELECT id FROM bitrix_sync.rebind_requests
+                    WHERE status IN ('pending','retry_wait')
+                    ORDER BY requested_at LIMIT :limit
+                    """
+                ),
+                {"limit": limit},
+            )
+            return list(rows.scalars())
+
+    async def reserve_limiter_token(self, refill_per_second: float, burst: int) -> float:
+        async with self.engine.begin() as connection:
+            row = (
+                await connection.execute(
+                    text(
+                        """
+                        SELECT tokens,capacity,refill_per_second,
+                               extract(epoch FROM now()-updated_at) elapsed,
+                               greatest(0,extract(epoch FROM blocked_until-now())) blocked
+                        FROM bitrix_sync.limiter_coordination
+                        WHERE limiter_key='bitrix24:portal'
+                        FOR UPDATE
+                        """
+                    )
+                )
+            ).mappings().first()
+            if row is None:
+                await connection.execute(
+                    text(
+                        """
+                        INSERT INTO bitrix_sync.limiter_coordination
+                          (limiter_key,tokens,capacity,refill_per_second,updated_at,fencing_token)
+                        VALUES ('bitrix24:portal',:tokens,:capacity,:refill,now(),1)
+                        """
+                    ),
+                    {"tokens": max(0, burst - 1), "capacity": burst, "refill": refill_per_second},
+                )
+                return 0
+            blocked = float(row["blocked"] or 0)
+            tokens = min(
+                float(burst),
+                float(row["tokens"]) + float(row["elapsed"] or 0) * refill_per_second,
+            )
+            if blocked > 0:
+                delay = blocked
+            elif tokens >= 1:
+                tokens -= 1
+                delay = 0
+            else:
+                delay = (1 - tokens) / refill_per_second
+            await connection.execute(
+                text(
+                    """
+                    UPDATE bitrix_sync.limiter_coordination
+                    SET tokens=:tokens,capacity=:capacity,refill_per_second=:refill,
+                        updated_at=now(),fencing_token=fencing_token+1
+                    WHERE limiter_key='bitrix24:portal'
+                    """
+                ),
+                {
+                    "tokens": tokens,
+                    "capacity": burst,
+                    "refill": refill_per_second,
+                },
+            )
+            return delay
+
+    async def status(self) -> dict[str, Any]:
+        queries = {
+            "queue": "SELECT status, count(*) count FROM han_app.sync_queue GROUP BY status",
+            "workflows": (
+                "SELECT state, count(*) count FROM bitrix_sync.workflow_instances GROUP BY state"
+            ),
+            "commands": (
+                "SELECT status, count(*) count "
+                "FROM bitrix_sync.crm_commands GROUP BY status"
+            ),
+            "webhook_lag_seconds": (
+                "SELECT coalesce(extract(epoch from now()-min(received_at)),0) "
+                "FROM bitrix_sync.webhook_inbox WHERE status IN ('received','retry_wait')"
+            ),
+            "settings_version": (
+                "SELECT version FROM bitrix_sync.settings_versions "
+                "WHERE active=true AND validation_status='valid' ORDER BY activated_at DESC LIMIT 1"
+            ),
+        }
+        output: dict[str, Any] = {}
+        async with self.engine.connect() as connection:
+            for key, sql in queries.items():
+                result = await connection.execute(text(sql))
+                if key in {"queue", "workflows", "commands"}:
+                    output[key] = {row.status: row.count for row in result}
+                else:
+                    output[key] = result.scalar_one_or_none()
+        output["generated_at"] = datetime.now(UTC).isoformat()
+        return output

VM2_services/codebase/services/bitrix-sync/app/security.py

@@ -0,0 +1,120 @@
+from __future__ import annotations
+
+import hmac
+import ipaddress
+import re
+from collections.abc import Mapping
+from dataclasses import dataclass
+from urllib.parse import parse_qsl
+
+from app.config import Settings
+
+SECRET_PATTERNS = (
+    re.compile(r"(?i)(token|authorization|password|secret)=([^&\s]+)"),
+    re.compile(r"https://[^/\s]+/rest/[0-9]+/[^/\s]+/"),
+)
+
+
+def token_matches(received: str | None, current: str, previous: str | None = None) -> bool:
+    candidate = (received or "").encode()
+    current_match = hmac.compare_digest(candidate, current.encode())
+    previous_match = hmac.compare_digest(candidate, (previous or "").encode())
+    return current_match or (previous is not None and previous_match)
+
+
+def redact(value: object) -> str:
+    text = str(value)
+    for pattern in SECRET_PATTERNS:
+        text = pattern.sub(
+            lambda match: (
+                f"{match.group(1)}=[REDACTED]"
+                if match.lastindex == 2
+                else "https://[REDACTED]/"
+            ),
+            text,
+        )
+    if "@" in text or re.search(r"\+7[0-9]{10}", text):
+        return "[PII_REDACTED]"
+    return text[:512]
+
+
+@dataclass(frozen=True)
+class WebhookEvent:
+    receiver_type: str
+    entity_id: str
+    event_type: str
+    source_ip: str
+
+
+class WebhookValidationError(ValueError):
+    pass
+
+
+def parse_bounded_form(body: bytes, *, max_fields: int) -> dict[str, str]:
+    try:
+        pairs = parse_qsl(body.decode("utf-8"), keep_blank_values=True, max_num_fields=max_fields)
+    except (UnicodeDecodeError, ValueError) as exc:
+        raise WebhookValidationError("malformed form") from exc
+    if len(pairs) > max_fields:
+        raise WebhookValidationError("too many form fields")
+    data: dict[str, str] = {}
+    for key, value in pairs:
+        if len(key) > 128 or len(value) > 512:
+            raise WebhookValidationError("form field too long")
+        data[key] = value
+    return data
+
+
+def validate_webhook(
+    receiver: str,
+    query: Mapping[str, str],
+    form: Mapping[str, str],
+    source_ip: str,
+    settings: Settings,
+    *,
+    alert_entity_type_id: int | None,
+) -> WebhookEvent:
+    try:
+        ip = ipaddress.ip_address(source_ip)
+    except ValueError as exc:
+        raise WebhookValidationError("invalid source address") from exc
+    if not any(ip in network for network in settings.allowed_networks):
+        raise PermissionError("source_ip")
+
+    configured = (
+        settings.contact_receiver_token if receiver == "contact" else settings.alert_receiver_token
+    )
+    previous = (
+        settings.contact_receiver_previous_token
+        if receiver == "contact"
+        else settings.alert_receiver_previous_token
+    )
+    if not configured or not token_matches(
+        query.get("token"),
+        configured.get_secret_value(),
+        previous.get_secret_value() if previous else None,
+    ):
+        raise PermissionError("token")
+    if form.get("auth[domain]", "").lower() != str(settings.portal_host).lower():
+        raise WebhookValidationError("portal mismatch")
+    if form.get("auth[member_id]") != settings.portal_member_id:
+        raise WebhookValidationError("member mismatch")
+    if form.get("document_id[0]") != "crm":
+        raise WebhookValidationError("invalid document module")
+
+    document_type = form.get("document_id[1]")
+    document_id = form.get("document_id[2]", "")
+    if receiver == "contact":
+        match = re.fullmatch(r"CONTACT_([1-9][0-9]*)", document_id)
+        if document_type != "CCrmDocumentContact" or not match:
+            raise WebhookValidationError("invalid contact document")
+    else:
+        match = re.fullmatch(r"DYNAMIC_([1-9][0-9]*)_([1-9][0-9]*)", document_id)
+        if not match or not document_type or "Dynamic" not in document_type:
+            raise WebhookValidationError("invalid alert document")
+        if alert_entity_type_id is None or int(match.group(1)) != alert_entity_type_id:
+            raise WebhookValidationError("alert entity type mismatch")
+    entity_id = match.group(match.lastindex or 1)
+    if query.get("ID") != entity_id:
+        raise WebhookValidationError("query/document ID mismatch")
+    return WebhookEvent(receiver, entity_id, f"{receiver}.changed", source_ip)

VM2_services/codebase/services/bitrix-sync/app/worker.py

@@ -0,0 +1,78 @@
+from __future__ import annotations
+
+import asyncio
+import signal
+import socket
+import uuid
+
+from app.config import load_settings
+from app.crm import CrmClient
+from app.domain import full_jitter_delay
+from app.engine import RetryableWorkflow, WorkflowEngine
+from app.repository import Repository
+
+
+async def worker_main() -> None:
+    settings = load_settings()
+    if not settings.enabled:
+        return
+    assert settings.database_url and settings.crm_rest_webhook_url and settings.portal_host
+    repository = Repository(settings.database_url.get_secret_value(), settings.db_pool_size)
+    crm = CrmClient(
+        settings.crm_rest_webhook_url.get_secret_value(),
+        settings.portal_host,
+        settings.http_timeout_sec,
+    )
+    engine = WorkflowEngine(repository, crm, settings)
+    stop = asyncio.Event()
+    loop = asyncio.get_running_loop()
+    for event in (signal.SIGINT, signal.SIGTERM):
+        try:
+            loop.add_signal_handler(event, stop.set)
+        except NotImplementedError:
+            pass
+    worker_id = f"{socket.gethostname()}:{uuid.uuid4()}"
+    try:
+        while not stop.is_set():
+            tasks = await repository.claim_tasks(
+                worker_id, settings.claim_size, settings.lease_seconds
+            )
+            webhooks = await repository.claim_webhooks(
+                worker_id, settings.claim_size, settings.lease_seconds
+            )
+            rebind_ids = await repository.pending_rebind_ids(settings.claim_size)
+            if not tasks and not webhooks and not rebind_ids:
+                try:
+                    await asyncio.wait_for(stop.wait(), timeout=1)
+                except TimeoutError:
+                    continue
+            for task in tasks:
+                if stop.is_set():
+                    break
+                await engine.process(task)
+            for item in webhooks:
+                if stop.is_set():
+                    break
+                try:
+                    await engine.process_webhook(item)
+                except RetryableWorkflow as exc:
+                    delay = exc.retry_after or full_jitter_delay(
+                        item.attempt_count + 1,
+                        settings.retry_base_seconds,
+                        settings.retry_max_seconds,
+                    )
+                    await repository.retry_webhook(item, exc.code, delay)
+            for request_id in rebind_ids:
+                if stop.is_set():
+                    break
+                try:
+                    await engine.process_rebind(request_id)
+                except RetryableWorkflow:
+                    continue
+    finally:
+        await crm.close()
+        await repository.close()
+
+
+def run() -> None:
+    asyncio.run(worker_main())