Текущая стабильная сборка с исправлением интеграционных косяков

2026-08-20 11:03:32 +03:00
parent c7a80e7256
commit 6e24278b10
12 changed files with 127 additions and 32 deletions
@@ -27,9 +27,9 @@ SELECTEL_S3_BUCKET_QUARANTINE=<quarantine-bucket>

 BITRIX_SYNC_ENABLED=false
 BITRIX_SYNC_MODE=disabled
-BITRIX_SYNC_CONTACT_USER_ID_FIELD=UF_CRM_<digits>
-BITRIX_SYNC_CONTACT_REGISTERED_FIELD=UF_CRM_<digits>
-BITRIX_SYNC_CONTACT_CITIZENSHIP_FIELD=UF_CRM_<digits>
+BITRIX_SYNC_CONTACT_USER_ID_FIELD=UF_CRM_<latin_letters_or_digits>
+BITRIX_SYNC_CONTACT_REGISTERED_FIELD=UF_CRM_<latin_letters_or_digits>
+BITRIX_SYNC_CONTACT_CITIZENSHIP_FIELD=UF_CRM_<latin_letters_or_digits>
 BITRIX_SYNC_PORTAL_HOST=<approved-portal>.bitrix24.ru
 BITRIX_SYNC_PORTAL_MEMBER_ID=<approved-member-id>
 BITRIX_SYNC_PUBLIC_BASE_URL=https://<processing-public-host>
@@ -15,4 +15,4 @@ COPY --chown=10001:10001 alembic.ini openapi.yaml /srv/
 WORKDIR /srv
 USER 10001:10001
 EXPOSE 8080
-ENTRYPOINT ["han-bitrix-sync-api"]
+CMD ["han-bitrix-sync-api"]
@@ -9,7 +9,7 @@ from urllib.parse import urlsplit
 from pydantic import Field, SecretStr, model_validator
 from pydantic_settings import BaseSettings, SettingsConfigDict

-FIELD_RE = re.compile(r"^UF_CRM_[0-9]+$")
+FIELD_RE = re.compile(r"^UF_CRM_[A-Za-z0-9]+$")
 MEMBER_RE = re.compile(r"^[A-Za-z0-9_-]{8,128}$")


@@ -107,7 +107,7 @@ class Settings(BaseSettings):
            "contact_citizenship_field",
        ):
            if not FIELD_RE.fullmatch(str(getattr(self, name))):
-                raise ValueError(f"{name} must match UF_CRM_<digits>")
+                raise ValueError(f"{name} must match UF_CRM_<latin letters or digits>")
        if not MEMBER_RE.fullmatch(str(self.portal_member_id)):
            raise ValueError("portal_member_id has invalid format")

@@ -5,7 +5,8 @@ import uuid
 from dataclasses import dataclass, field
 from typing import Any

-from sqlalchemy import text
+from sqlalchemy import bindparam, text
+from sqlalchemy.dialects.postgresql import JSONB

 from app.config import Settings
 from app.crm import CrmClient, CrmOutcome, CrmResult
@@ -19,6 +20,26 @@ from app.domain import (
 )
 from app.repository import LeasedTask, LeasedWebhook, Profile, Repository

+INSERT_CRM_COMMAND = text(
+    """
+    INSERT INTO bitrix_sync.crm_commands
+      (id,workflow_id,command_type,safe_request,status,attempt_count,
+       next_attempt_at,created_at,updated_at)
+    VALUES (:id,:workflow_id,:type,:safe_request,'in_flight',1,now(),now(),now())
+    """
+).bindparams(bindparam("safe_request", type_=JSONB()))
+
+UPDATE_CRM_COMMAND = text(
+    """
+    UPDATE bitrix_sync.crm_commands
+    SET status=:status,safe_error_code=:error,http_status=:http_status,
+        safe_response=:safe_response,
+        completed_at=CASE WHEN :terminal THEN now() END,
+        updated_at=now()
+    WHERE id=:id
+    """
+).bindparams(bindparam("safe_response", type_=JSONB()))
+

 class BusinessConflict(Exception):
    def __init__(self, code: str, candidates: list[str] | None = None) -> None:
@@ -452,14 +473,7 @@ class WorkflowEngine:
        command_id = uuid.uuid4()
        async with self.repository.transaction() as connection:
            await connection.execute(
-                text(
-                    """
-                    INSERT INTO bitrix_sync.crm_commands
-                      (id,workflow_id,command_type,safe_request,status,attempt_count,
-                       next_attempt_at,created_at,updated_at)
-                    VALUES (:id,:workflow_id,:type,:safe_request,'in_flight',1,now(),now(),now())
-                    """
-                ),
+                INSERT_CRM_COMMAND,
                {
                    "id": command_id,
                    "workflow_id": workflow_id,
@@ -479,16 +493,7 @@ class WorkflowEngine:
        status = result.outcome.value
        async with self.repository.transaction() as connection:
            await connection.execute(
-                text(
-                    """
-                    UPDATE bitrix_sync.crm_commands
-                    SET status=:status,safe_error_code=:error,http_status=:http_status,
-                        safe_response=:safe_response,
-                        completed_at=CASE WHEN :terminal THEN now() END,
-                        updated_at=now()
-                    WHERE id=:id
-                    """
-                ),
+                UPDATE_CRM_COMMAND,
                {
                    "id": command_id,
                    "status": status,
@@ -506,7 +506,8 @@ class Repository:
        queries = {
            "queue": "SELECT status, count(*) count FROM han_app.sync_queue GROUP BY status",
            "workflows": (
-                "SELECT state, count(*) count FROM bitrix_sync.workflow_instances GROUP BY state"
+                "SELECT state AS status, count(*) count "
+                "FROM bitrix_sync.workflow_instances GROUP BY state"
            ),
            "commands": (
                "SELECT status, count(*) count "
@@ -37,8 +37,25 @@ def test_full_mode_rejects_portal_host_mismatch() -> None:

 def test_rest_field_conversion_is_deterministic() -> None:
    assert Settings.rest_field_name("UF_CRM_1778692456") == "ufCrm_1778692456"
+    assert Settings.rest_field_name("UF_CRM_6a70c275346a7") == "ufCrm_6a70c275346a7"
+    assert Settings.rest_field_name("UF_CRM_AbC123") == "ufCrm_AbC123"
    with pytest.raises(ValueError):
        Settings.rest_field_name("uf_crm_1")
+    with pytest.raises(ValueError):
+        Settings.rest_field_name("UF_CRM_123_abc")
+
+
+def test_image_default_allows_compose_process_role_override() -> None:
+    service_root = Path(__file__).parents[1]
+    dockerfile = (service_root / "Dockerfile").read_text(encoding="utf-8")
+    compose = (service_root / "compose.fragment.yaml").read_text(encoding="utf-8")
+    production_compose = (service_root.parent / "docker-compose.yml").read_text(encoding="utf-8")
+
+    assert 'CMD ["han-bitrix-sync-api"]' in dockerfile
+    assert 'ENTRYPOINT ["han-bitrix-sync-api"]' not in dockerfile
+    for source in (compose, production_compose):
+        assert 'command: ["han-bitrix-sync-worker"]' in source
+        assert 'command: ["han-bitrix-sync-reconciliation"]' in source


 def test_alembic_chain_preserves_legacy_baseline() -> None:
@@ -4,9 +4,10 @@ import uuid
 from contextlib import asynccontextmanager

 import pytest
+from sqlalchemy.dialects.postgresql import JSONB

 from app.crm import CrmOutcome, CrmResult
-from app.engine import WorkflowEngine
+from app.engine import INSERT_CRM_COMMAND, UPDATE_CRM_COMMAND, WorkflowEngine
 from app.repository import Profile


@@ -57,6 +58,11 @@ class FakeCrm:
        return CrmResult(CrmOutcome.SUCCEEDED, True)


+def test_crm_command_json_payloads_have_explicit_jsonb_types() -> None:
+    assert isinstance(INSERT_CRM_COMMAND._bindparams["safe_request"].type, JSONB)
+    assert isinstance(UPDATE_CRM_COMMAND._bindparams["safe_response"].type, JSONB)
+
+
@pytest.mark.asyncio
 async def test_multiple_contacts_choose_numeric_newest(full_settings) -> None:
    user_id = uuid.uuid4()
@@ -0,0 +1,57 @@
+from __future__ import annotations
+
+from contextlib import asynccontextmanager
+from types import SimpleNamespace
+
+import pytest
+
+from app.repository import Repository
+
+
+class FakeResult:
+    def __init__(self, *, rows=(), scalar=None) -> None:
+        self.rows = rows
+        self.scalar = scalar
+
+    def __iter__(self):
+        return iter(self.rows)
+
+    def scalar_one_or_none(self):
+        return self.scalar
+
+
+class FakeConnection:
+    async def execute(self, statement):
+        sql = str(statement)
+        if "han_app.sync_queue" in sql:
+            return FakeResult(rows=[SimpleNamespace(status="pending", count=2)])
+        if "workflow_instances" in sql:
+            assert "state AS status" in sql
+            return FakeResult(rows=[SimpleNamespace(status="created", count=3)])
+        if "crm_commands" in sql:
+            return FakeResult(rows=[SimpleNamespace(status="succeeded", count=4)])
+        if "webhook_inbox" in sql:
+            return FakeResult(scalar=1.5)
+        if "settings_versions" in sql:
+            return FakeResult(scalar=1)
+        raise AssertionError(f"unexpected status query: {sql}")
+
+
+class FakeEngine:
+    @asynccontextmanager
+    async def connect(self):
+        yield FakeConnection()
+
+
+@pytest.mark.asyncio
+async def test_status_uses_common_status_alias_for_workflows() -> None:
+    repository = object.__new__(Repository)
+    repository.engine = FakeEngine()
+
+    result = await repository.status()
+
+    assert result["queue"] == {"pending": 2}
+    assert result["workflows"] == {"created": 3}
+    assert result["commands"] == {"succeeded": 4}
+    assert result["webhook_lag_seconds"] == 1.5
+    assert result["settings_version"] == 1
@@ -71,7 +71,7 @@ BITRIX_SYNC_CONTACT_CITIZENSHIP_FIELD=UF_CRM_1768493029

 Значения обязательны при `BITRIX_SYNC_ENABLED=true`, проверяются на startup по допустимому формату имени поля. Изменение требует контролируемого restart.

-Для методов universal CRM физическое имя `UF_CRM_<digits>` детерминированно преобразуется в REST-имя `ufCrm_<digits>`. В частности, `BITRIX_SYNC_CONTACT_REGISTERED_FIELD=UF_CRM_1778692456` используется в `crm.item.list` как `ufCrm_1778692456`. Произвольное изменение регистра или иной алгоритм преобразования запрещены; обе формы валидируются на startup.
+Для методов universal CRM физическое имя `UF_CRM_<latin letters or digits>` детерминированно преобразуется в REST-имя `ufCrm_<latin letters or digits>`. В частности, `BITRIX_SYNC_CONTACT_REGISTERED_FIELD=UF_CRM_6a70c275346a7` используется в `crm.item.list` как `ufCrm_6a70c275346a7`. Произвольное изменение регистра или иной алгоритм преобразования запрещены; обе формы валидируются на startup.

 ### 3.3. Преобразования

@@ -81,7 +81,7 @@ Config-only rollout — отдельный root-owned job под config-admin ro

 До `BITRIX_SYNC_ENABLED=true`:

-1. custom Contact fields и env names; universal CRM `UF_CRM_<digits>` → `ufCrm_<digits>`;
+1. custom Contact fields и env names; universal CRM `UF_CRM_<latin letters or digits>` → `ufCrm_<latin letters or digits>`;
 2. smart process конфликтов и `bitrix_sync.settings`;
 3. входящий webhook техпользователя, права module-07 §13;
 4. два HTTP-webhook на `https://<PROCESSING_PUBLIC_HOST>/bitrix/sync/webhook/...?token=...`; query/body не в журналах; local app handler не создавать;
@@ -59,7 +59,7 @@
 12. #INFRASTRUCTURE Перераскатить сервисы от деплоя
 13. #INFRASTRUCTURE Поднять второй контур для продакшн   
 14. #INFRASTRUCTURE Спрятать сеть за балансировщиком нагрузки
-
+15. #UI Добавить подсказку по разрешенным типам файлов
 16. #INFRASTRUCTURE WireGuard-only SSH.
 17. #LEGAL Обновить документы по ПД - модель угроз и меры защиты.
 18. #LEGAL Уведомление в РКН по БД обработки ПД.
@@ -76,10 +76,11 @@
 29. #BACK_BUSINESS Определить пул тестовых номеров, чтобы их было легко в Б24 отслеживать.
 30. #INFRASTRUCTURE перевести взаимодействие с signoz на TLS (сейчас OTEL_REMOTE_TLS_INSECURE=true)
 31. #BACK_BUSINESS VM1 -> VM2: `curl -sS --cacert /etc/han/ca/vm2-internal-ca.crt   "https://processing.internal:8443/internal/safety/status" | python3 -m json.tool` В коде захардкожен Redis в components = "degraded". Надо реализовать реальную проверку вместо костыля.
+32. #BACK_BUSINESS Решить проблему с обновлением сигнатур CLAMAV.

 # Критично для релиза:
-1. Разработка message-safety
-2. Разработка sync-service
+~~1. Разработка message-safety~~
+~~2. Разработка sync-service~~
 3. Пользовательское соглашение
 ~~4. Подключить OTLP-провайдер~~
 ~~5. Починить UI баги~~
@@ -0,0 +1,8 @@
+#-B10: показать 10 строк до найденного значения
+#-A40: показать 40 строк после найденного значения
+
+/usr/local/sbin/han-vm1-compose logs --since=30m api-backend \
+  | grep -B10 -A40 'bfffb1c2-b938-4443-9c42-5a9b87fcf6b3'
+
+/usr/local/sbin/han-vm1-compose logs --since=30m api-backend \
+  | grep -B5 -A20 'request.failed'