diff --git a/VM2_services/codebase/services/.env.example b/VM2_services/codebase/services/.env.example index 47deab1..3e8db80 100644 --- a/VM2_services/codebase/services/.env.example +++ b/VM2_services/codebase/services/.env.example @@ -27,9 +27,9 @@ SELECTEL_S3_BUCKET_QUARANTINE= BITRIX_SYNC_ENABLED=false BITRIX_SYNC_MODE=disabled -BITRIX_SYNC_CONTACT_USER_ID_FIELD=UF_CRM_ -BITRIX_SYNC_CONTACT_REGISTERED_FIELD=UF_CRM_ -BITRIX_SYNC_CONTACT_CITIZENSHIP_FIELD=UF_CRM_ +BITRIX_SYNC_CONTACT_USER_ID_FIELD=UF_CRM_ +BITRIX_SYNC_CONTACT_REGISTERED_FIELD=UF_CRM_ +BITRIX_SYNC_CONTACT_CITIZENSHIP_FIELD=UF_CRM_ BITRIX_SYNC_PORTAL_HOST=.bitrix24.ru BITRIX_SYNC_PORTAL_MEMBER_ID= BITRIX_SYNC_PUBLIC_BASE_URL=https:// diff --git a/VM2_services/codebase/services/bitrix-sync/Dockerfile b/VM2_services/codebase/services/bitrix-sync/Dockerfile index 0fe6996..e9a52db 100644 --- a/VM2_services/codebase/services/bitrix-sync/Dockerfile +++ b/VM2_services/codebase/services/bitrix-sync/Dockerfile @@ -15,4 +15,4 @@ COPY --chown=10001:10001 alembic.ini openapi.yaml /srv/ WORKDIR /srv USER 10001:10001 EXPOSE 8080 -ENTRYPOINT ["han-bitrix-sync-api"] +CMD ["han-bitrix-sync-api"] diff --git a/VM2_services/codebase/services/bitrix-sync/app/config.py b/VM2_services/codebase/services/bitrix-sync/app/config.py index 9915957..0619bf1 100644 --- a/VM2_services/codebase/services/bitrix-sync/app/config.py +++ b/VM2_services/codebase/services/bitrix-sync/app/config.py @@ -9,7 +9,7 @@ from urllib.parse import urlsplit from pydantic import Field, SecretStr, model_validator from pydantic_settings import BaseSettings, SettingsConfigDict -FIELD_RE = re.compile(r"^UF_CRM_[0-9]+$") +FIELD_RE = re.compile(r"^UF_CRM_[A-Za-z0-9]+$") MEMBER_RE = re.compile(r"^[A-Za-z0-9_-]{8,128}$") @@ -107,7 +107,7 @@ class Settings(BaseSettings): "contact_citizenship_field", ): if not FIELD_RE.fullmatch(str(getattr(self, name))): - raise ValueError(f"{name} must match UF_CRM_") + raise ValueError(f"{name} must match UF_CRM_") if not MEMBER_RE.fullmatch(str(self.portal_member_id)): raise ValueError("portal_member_id has invalid format") diff --git a/VM2_services/codebase/services/bitrix-sync/app/engine.py b/VM2_services/codebase/services/bitrix-sync/app/engine.py index 7cdcab8..058f8db 100644 --- a/VM2_services/codebase/services/bitrix-sync/app/engine.py +++ b/VM2_services/codebase/services/bitrix-sync/app/engine.py @@ -5,7 +5,8 @@ import uuid from dataclasses import dataclass, field from typing import Any -from sqlalchemy import text +from sqlalchemy import bindparam, text +from sqlalchemy.dialects.postgresql import JSONB from app.config import Settings from app.crm import CrmClient, CrmOutcome, CrmResult @@ -19,6 +20,26 @@ from app.domain import ( ) from app.repository import LeasedTask, LeasedWebhook, Profile, Repository +INSERT_CRM_COMMAND = text( + """ + INSERT INTO bitrix_sync.crm_commands + (id,workflow_id,command_type,safe_request,status,attempt_count, + next_attempt_at,created_at,updated_at) + VALUES (:id,:workflow_id,:type,:safe_request,'in_flight',1,now(),now(),now()) + """ +).bindparams(bindparam("safe_request", type_=JSONB())) + +UPDATE_CRM_COMMAND = text( + """ + UPDATE bitrix_sync.crm_commands + SET status=:status,safe_error_code=:error,http_status=:http_status, + safe_response=:safe_response, + completed_at=CASE WHEN :terminal THEN now() END, + updated_at=now() + WHERE id=:id + """ +).bindparams(bindparam("safe_response", type_=JSONB())) + class BusinessConflict(Exception): def __init__(self, code: str, candidates: list[str] | None = None) -> None: @@ -452,14 +473,7 @@ class WorkflowEngine: command_id = uuid.uuid4() async with self.repository.transaction() as connection: await connection.execute( - text( - """ - INSERT INTO bitrix_sync.crm_commands - (id,workflow_id,command_type,safe_request,status,attempt_count, - next_attempt_at,created_at,updated_at) - VALUES (:id,:workflow_id,:type,:safe_request,'in_flight',1,now(),now(),now()) - """ - ), + INSERT_CRM_COMMAND, { "id": command_id, "workflow_id": workflow_id, @@ -479,16 +493,7 @@ class WorkflowEngine: status = result.outcome.value async with self.repository.transaction() as connection: await connection.execute( - text( - """ - UPDATE bitrix_sync.crm_commands - SET status=:status,safe_error_code=:error,http_status=:http_status, - safe_response=:safe_response, - completed_at=CASE WHEN :terminal THEN now() END, - updated_at=now() - WHERE id=:id - """ - ), + UPDATE_CRM_COMMAND, { "id": command_id, "status": status, diff --git a/VM2_services/codebase/services/bitrix-sync/app/repository.py b/VM2_services/codebase/services/bitrix-sync/app/repository.py index 5c6822d..a452e9c 100644 --- a/VM2_services/codebase/services/bitrix-sync/app/repository.py +++ b/VM2_services/codebase/services/bitrix-sync/app/repository.py @@ -506,7 +506,8 @@ class Repository: queries = { "queue": "SELECT status, count(*) count FROM han_app.sync_queue GROUP BY status", "workflows": ( - "SELECT state, count(*) count FROM bitrix_sync.workflow_instances GROUP BY state" + "SELECT state AS status, count(*) count " + "FROM bitrix_sync.workflow_instances GROUP BY state" ), "commands": ( "SELECT status, count(*) count " diff --git a/VM2_services/codebase/services/bitrix-sync/tests/test_config.py b/VM2_services/codebase/services/bitrix-sync/tests/test_config.py index 6ef7f91..70a7cd9 100644 --- a/VM2_services/codebase/services/bitrix-sync/tests/test_config.py +++ b/VM2_services/codebase/services/bitrix-sync/tests/test_config.py @@ -37,8 +37,25 @@ def test_full_mode_rejects_portal_host_mismatch() -> None: def test_rest_field_conversion_is_deterministic() -> None: assert Settings.rest_field_name("UF_CRM_1778692456") == "ufCrm_1778692456" + assert Settings.rest_field_name("UF_CRM_6a70c275346a7") == "ufCrm_6a70c275346a7" + assert Settings.rest_field_name("UF_CRM_AbC123") == "ufCrm_AbC123" with pytest.raises(ValueError): Settings.rest_field_name("uf_crm_1") + with pytest.raises(ValueError): + Settings.rest_field_name("UF_CRM_123_abc") + + +def test_image_default_allows_compose_process_role_override() -> None: + service_root = Path(__file__).parents[1] + dockerfile = (service_root / "Dockerfile").read_text(encoding="utf-8") + compose = (service_root / "compose.fragment.yaml").read_text(encoding="utf-8") + production_compose = (service_root.parent / "docker-compose.yml").read_text(encoding="utf-8") + + assert 'CMD ["han-bitrix-sync-api"]' in dockerfile + assert 'ENTRYPOINT ["han-bitrix-sync-api"]' not in dockerfile + for source in (compose, production_compose): + assert 'command: ["han-bitrix-sync-worker"]' in source + assert 'command: ["han-bitrix-sync-reconciliation"]' in source def test_alembic_chain_preserves_legacy_baseline() -> None: diff --git a/VM2_services/codebase/services/bitrix-sync/tests/test_engine_boundaries.py b/VM2_services/codebase/services/bitrix-sync/tests/test_engine_boundaries.py index 845cf1f..1b98d7a 100644 --- a/VM2_services/codebase/services/bitrix-sync/tests/test_engine_boundaries.py +++ b/VM2_services/codebase/services/bitrix-sync/tests/test_engine_boundaries.py @@ -4,9 +4,10 @@ import uuid from contextlib import asynccontextmanager import pytest +from sqlalchemy.dialects.postgresql import JSONB from app.crm import CrmOutcome, CrmResult -from app.engine import WorkflowEngine +from app.engine import INSERT_CRM_COMMAND, UPDATE_CRM_COMMAND, WorkflowEngine from app.repository import Profile @@ -57,6 +58,11 @@ class FakeCrm: return CrmResult(CrmOutcome.SUCCEEDED, True) +def test_crm_command_json_payloads_have_explicit_jsonb_types() -> None: + assert isinstance(INSERT_CRM_COMMAND._bindparams["safe_request"].type, JSONB) + assert isinstance(UPDATE_CRM_COMMAND._bindparams["safe_response"].type, JSONB) + + @pytest.mark.asyncio async def test_multiple_contacts_choose_numeric_newest(full_settings) -> None: user_id = uuid.uuid4() diff --git a/VM2_services/codebase/services/bitrix-sync/tests/test_repository.py b/VM2_services/codebase/services/bitrix-sync/tests/test_repository.py new file mode 100644 index 0000000..8e02b49 --- /dev/null +++ b/VM2_services/codebase/services/bitrix-sync/tests/test_repository.py @@ -0,0 +1,57 @@ +from __future__ import annotations + +from contextlib import asynccontextmanager +from types import SimpleNamespace + +import pytest + +from app.repository import Repository + + +class FakeResult: + def __init__(self, *, rows=(), scalar=None) -> None: + self.rows = rows + self.scalar = scalar + + def __iter__(self): + return iter(self.rows) + + def scalar_one_or_none(self): + return self.scalar + + +class FakeConnection: + async def execute(self, statement): + sql = str(statement) + if "han_app.sync_queue" in sql: + return FakeResult(rows=[SimpleNamespace(status="pending", count=2)]) + if "workflow_instances" in sql: + assert "state AS status" in sql + return FakeResult(rows=[SimpleNamespace(status="created", count=3)]) + if "crm_commands" in sql: + return FakeResult(rows=[SimpleNamespace(status="succeeded", count=4)]) + if "webhook_inbox" in sql: + return FakeResult(scalar=1.5) + if "settings_versions" in sql: + return FakeResult(scalar=1) + raise AssertionError(f"unexpected status query: {sql}") + + +class FakeEngine: + @asynccontextmanager + async def connect(self): + yield FakeConnection() + + +@pytest.mark.asyncio +async def test_status_uses_common_status_alias_for_workflows() -> None: + repository = object.__new__(Repository) + repository.engine = FakeEngine() + + result = await repository.status() + + assert result["queue"] == {"pending": 2} + assert result["workflows"] == {"created": 3} + assert result["commands"] == {"succeeded": 4} + assert result["webhook_lag_seconds"] == 1.5 + assert result["settings_version"] == 1 diff --git a/VM2_services/documentation/module-07-bitrix-sync.md b/VM2_services/documentation/module-07-bitrix-sync.md index 6f846e1..0aba4d0 100644 --- a/VM2_services/documentation/module-07-bitrix-sync.md +++ b/VM2_services/documentation/module-07-bitrix-sync.md @@ -71,7 +71,7 @@ BITRIX_SYNC_CONTACT_CITIZENSHIP_FIELD=UF_CRM_1768493029 Значения обязательны при `BITRIX_SYNC_ENABLED=true`, проверяются на startup по допустимому формату имени поля. Изменение требует контролируемого restart. -Для методов universal CRM физическое имя `UF_CRM_` детерминированно преобразуется в REST-имя `ufCrm_`. В частности, `BITRIX_SYNC_CONTACT_REGISTERED_FIELD=UF_CRM_1778692456` используется в `crm.item.list` как `ufCrm_1778692456`. Произвольное изменение регистра или иной алгоритм преобразования запрещены; обе формы валидируются на startup. +Для методов universal CRM физическое имя `UF_CRM_` детерминированно преобразуется в REST-имя `ufCrm_`. В частности, `BITRIX_SYNC_CONTACT_REGISTERED_FIELD=UF_CRM_6a70c275346a7` используется в `crm.item.list` как `ufCrm_6a70c275346a7`. Произвольное изменение регистра или иной алгоритм преобразования запрещены; обе формы валидируются на startup. ### 3.3. Преобразования diff --git a/VM2_services/documentation/module-10-deployment-vm2.md b/VM2_services/documentation/module-10-deployment-vm2.md index 064ee39..1b7ad45 100644 --- a/VM2_services/documentation/module-10-deployment-vm2.md +++ b/VM2_services/documentation/module-10-deployment-vm2.md @@ -81,7 +81,7 @@ Config-only rollout — отдельный root-owned job под config-admin ro До `BITRIX_SYNC_ENABLED=true`: -1. custom Contact fields и env names; universal CRM `UF_CRM_` → `ufCrm_`; +1. custom Contact fields и env names; universal CRM `UF_CRM_` → `ufCrm_`; 2. smart process конфликтов и `bitrix_sync.settings`; 3. входящий webhook техпользователя, права module-07 §13; 4. два HTTP-webhook на `https:///bitrix/sync/webhook/...?token=...`; query/body не в журналах; local app handler не создавать; diff --git a/support¬es/backlog.md b/support¬es/backlog.md index 5d2ae37..ae75d5d 100644 --- a/support¬es/backlog.md +++ b/support¬es/backlog.md @@ -59,7 +59,7 @@ 12. #INFRASTRUCTURE Перераскатить сервисы от деплоя 13. #INFRASTRUCTURE Поднять второй контур для продакшн 14. #INFRASTRUCTURE Спрятать сеть за балансировщиком нагрузки - +15. #UI Добавить подсказку по разрешенным типам файлов 16. #INFRASTRUCTURE WireGuard-only SSH. 17. #LEGAL Обновить документы по ПД - модель угроз и меры защиты. 18. #LEGAL Уведомление в РКН по БД обработки ПД. @@ -76,10 +76,11 @@ 29. #BACK_BUSINESS Определить пул тестовых номеров, чтобы их было легко в Б24 отслеживать. 30. #INFRASTRUCTURE перевести взаимодействие с signoz на TLS (сейчас OTEL_REMOTE_TLS_INSECURE=true) 31. #BACK_BUSINESS VM1 -> VM2: `curl -sS --cacert /etc/han/ca/vm2-internal-ca.crt "https://processing.internal:8443/internal/safety/status" | python3 -m json.tool` В коде захардкожен Redis в components = "degraded". Надо реализовать реальную проверку вместо костыля. +32. #BACK_BUSINESS Решить проблему с обновлением сигнатур CLAMAV. # Критично для релиза: -1. Разработка message-safety -2. Разработка sync-service +~~1. Разработка message-safety~~ +~~2. Разработка sync-service~~ 3. Пользовательское соглашение ~~4. Подключить OTLP-провайдер~~ ~~5. Починить UI баги~~ diff --git a/support¬es/usefull ops/diagnostic_scripts.md b/support¬es/usefull ops/diagnostic_scripts.md index e69de29..5cbebe6 100644 --- a/support¬es/usefull ops/diagnostic_scripts.md +++ b/support¬es/usefull ops/diagnostic_scripts.md @@ -0,0 +1,8 @@ +#-B10: показать 10 строк до найденного значения +#-A40: показать 40 строк после найденного значения + +/usr/local/sbin/han-vm1-compose logs --since=30m api-backend \ + | grep -B10 -A40 'bfffb1c2-b938-4443-9c42-5a9b87fcf6b3' + +/usr/local/sbin/han-vm1-compose logs --since=30m api-backend \ + | grep -B5 -A20 'request.failed' \ No newline at end of file