Admin/han-app
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Финальная стабильная реализация

Browse Source
This commit is contained in:
mi
2026-08-21 11:48:24 +03:00
parent 6e24278b10
commit d2415fcfeb
17 changed files with 662 additions and 57 deletions
Download Patch File Download Diff File Expand all files Collapse all files
VM2_services/codebase/services/.env.example
+1
View File
@@ -30,6 +30,7 @@ BITRIX_SYNC_MODE=disabled
BITRIX_SYNC_CONTACT_USER_ID_FIELD=UF_CRM_<latin_letters_or_digits>
BITRIX_SYNC_CONTACT_REGISTERED_FIELD=UF_CRM_<latin_letters_or_digits>
BITRIX_SYNC_CONTACT_CITIZENSHIP_FIELD=UF_CRM_<latin_letters_or_digits>
BITRIX_SYNC_CONTACT_SOURCE=<bitrix-contact-source-id>
BITRIX_SYNC_PORTAL_HOST=<approved-portal>.bitrix24.ru
BITRIX_SYNC_PORTAL_MEMBER_ID=<approved-member-id>
BITRIX_SYNC_PUBLIC_BASE_URL=https://<processing-public-host>
VM2_services/codebase/services/bitrix-sync/.env.example
+1
View File
@@ -11,6 +11,7 @@ BITRIX_SYNC_PUBLIC_BASE_URL=https://sync.example.ru
BITRIX_SYNC_CONTACT_USER_ID_FIELD=UF_CRM_100
BITRIX_SYNC_CONTACT_REGISTERED_FIELD=UF_CRM_101
BITRIX_SYNC_CONTACT_CITIZENSHIP_FIELD=UF_CRM_102
BITRIX_SYNC_CONTACT_SOURCE=WEB
BITRIX_SYNC_WEBHOOK_ALLOWED_CIDRS=203.0.113.0/24
BITRIX_SYNC_HTTP_TIMEOUT_SEC=10
BITRIX_SYNC_DB_POOL_SIZE=5
VM2_services/codebase/services/bitrix-sync/README.md
+9
View File
@@ -54,6 +54,15 @@ gates: локальный managed PostgreSQL не поднимается Compose
4. Заполнить и активировать валидную `business_alerts` settings version:
entity/category/stage/field IDs и responsible party. Placeholder `null`
запрещает alert receiver.
`value_json` настройки `business_alerts` использует ключи `entity_type_id`,
`category_id`, `stage_new`, опциональный `responsible_id` и объект `field_ids`.
В `field_ids` REST-имена пользовательских полей сопоставляются ключам
`alert_number`, `fingerprint`, `alert_type`, `severity`, `app_user_id`,
`selected_external_id`, `occurrence_count`,
`first_occurred_at`, `last_occurred_at`, `workflow_id`.
Конфликтующие Contact передаются в стандартное поле `contactIds`, а значение
`BITRIX_SYNC_CONTACT_SOURCE` — в стандартное поле `sourceId`.
5. Проверить least-privilege credential negative tests; credential администратора
запрещён.
6. Валидировать nginx exact routes, no-redirect HTTP policy, body/rate limits,
VM2_services/codebase/services/bitrix-sync/app/config.py
+6
View File
@@ -46,6 +46,7 @@ class Settings(BaseSettings):
contact_user_id_field: str | None = None
contact_registered_field: str | None = None
contact_citizenship_field: str | None = None
contact_source: str | None = Field(default=None, min_length=1, max_length=128)
webhook_allowed_cidrs: str = ""
http_timeout_sec: float = Field(default=10, ge=1, le=60)
@@ -97,6 +98,7 @@ class Settings(BaseSettings):
"contact_user_id_field": self.contact_user_id_field,
"contact_registered_field": self.contact_registered_field,
"contact_citizenship_field": self.contact_citizenship_field,
"contact_source": self.contact_source,
}
missing = [name for name, value in required.items() if not value]
if missing:
@@ -110,6 +112,10 @@ class Settings(BaseSettings):
raise ValueError(f"{name} must match UF_CRM_<latin letters or digits>")
if not MEMBER_RE.fullmatch(str(self.portal_member_id)):
raise ValueError("portal_member_id has invalid format")
assert self.contact_source
self.contact_source = self.contact_source.strip()
if not self.contact_source:
raise ValueError("contact_source cannot be blank")
crm = urlsplit(self.crm_rest_webhook_url.get_secret_value())
public = urlsplit(str(self.public_base_url))
VM2_services/codebase/services/bitrix-sync/app/engine.py
+155 -21
View File
@@ -15,6 +15,7 @@ from app.domain import (
choose_newest,
full_jitter_delay,
parse_crm_datetime,
safe_hash,
select_email,
validate_phone,
)
@@ -199,6 +200,14 @@ class WorkflowEngine:
alert_code: str | None = None
if same_user:
selected_id = str(same_user["ID"])
if len(contacts) > 1:
await self._alert(
workflow_id,
profile.user_id,
"duplicate_contacts",
[str(item["ID"]) for item in contacts],
selected_external_id=selected_id,
)
elif not contacts:
selected_id = await self._create_contact(workflow_id, profile)
else:
@@ -214,15 +223,27 @@ class WorkflowEngine:
assert selected is not None
all_ids = [item.b24_id for item in candidates]
if selected.crm_user_id and selected.crm_user_id != str(profile.user_id):
selected_id = await self._create_contact(workflow_id, profile)
alert_code = "contact_owned_by_other_user"
selected_id = await self._create_contact(workflow_id, profile)
await self._alert(
workflow_id,
profile.user_id,
alert_code,
[*all_ids, selected_id],
selected_external_id=selected_id,
)
else:
selected_id = selected.b24_id
await self._write_identity(workflow_id, selected_id, profile.user_id, active=True)
if len(candidates) > 1:
alert_code = "duplicate_contacts"
if alert_code:
await self._alert(workflow_id, profile.user_id, alert_code, all_ids)
await self._alert(
workflow_id,
profile.user_id,
alert_code,
all_ids,
selected_external_id=selected_id,
)
await self._write_identity(workflow_id, selected_id, profile.user_id, active=True)
await self._activate_mapping(workflow_id, profile.user_id, selected_id)
async def _create_contact(self, workflow_id: uuid.UUID, profile: Profile) -> str:
@@ -551,31 +572,144 @@ class WorkflowEngine:
)
async def _alert(
self, workflow_id: uuid.UUID, user_id: uuid.UUID, alert_type: str, candidates: list[str]
self,
workflow_id: uuid.UUID,
user_id: uuid.UUID,
alert_type: str,
candidates: list[str],
*,
selected_external_id: str | None = None,
) -> None:
fingerprint = f"{alert_type}:{user_id}"
fingerprint = safe_hash(f"{alert_type}:{user_id}")
assert fingerprint is not None
async with self.repository.transaction() as connection:
alert = (
await connection.execute(
text(
"""
INSERT INTO bitrix_sync.business_alerts
(id,fingerprint,alert_type,severity,app_user_id,
selected_external_id,candidate_external_ids,workflow_id,status,
occurrence_count,first_occurred_at,last_occurred_at,created_at,updated_at)
VALUES (gen_random_uuid(),:fingerprint,
:type,'warning',:user_id,:selected_external_id,:candidates,
:workflow_id,'open',1,now(),now(),now(),now())
ON CONFLICT (alert_type,fingerprint) WHERE status='open'
DO UPDATE SET
selected_external_id=coalesce(
excluded.selected_external_id,
business_alerts.selected_external_id
),
candidate_external_ids=excluded.candidate_external_ids,
workflow_id=excluded.workflow_id,
occurrence_count=business_alerts.occurrence_count+1,
last_occurred_at=now(),updated_at=now()
RETURNING id,alert_number,fingerprint,alert_type,severity,app_user_id,
selected_external_id,candidate_external_ids,remote_item_id,
occurrence_count,first_occurred_at,last_occurred_at
"""
),
{
"fingerprint": fingerprint,
"type": alert_type,
"user_id": user_id,
"selected_external_id": selected_external_id,
"candidates": candidates,
"workflow_id": workflow_id,
},
)
).mappings().one()
alert_config = (
await connection.execute(
text(
"""
SELECT value_json FROM bitrix_sync.settings
WHERE key='business_alerts' AND active=true
AND validation_status='valid'
"""
)
)
).scalar_one_or_none()
if not isinstance(alert_config, dict):
raise RetryableWorkflow("business_alerts_not_configured")
entity_type_id = alert_config.get("entity_type_id")
stage_new = alert_config.get("stage_new")
if not entity_type_id or not stage_new:
raise RetryableWorkflow("business_alerts_not_configured")
title = (
f"Конфликт синхронизации #{alert['alert_number']}: {alert_type}; "
f"user={user_id}; contacts={','.join(candidates)}"
)
fields: dict[str, Any] = {
"title": title[:255],
"stageId": stage_new,
"contactIds": [int(candidate) for candidate in candidates if candidate.isdigit()],
"sourceId": self.settings.contact_source,
}
if category_id := alert_config.get("category_id"):
fields["categoryId"] = category_id
if responsible_id := alert_config.get("responsible_id"):
fields["assignedById"] = responsible_id
alert_values = {
"alert_number": str(alert["alert_number"]),
"fingerprint": alert["fingerprint"],
"alert_type": alert["alert_type"],
"severity": alert["severity"],
"app_user_id": str(alert["app_user_id"]),
"selected_external_id": alert["selected_external_id"] or "",
"occurrence_count": alert["occurrence_count"],
"first_occurred_at": alert["first_occurred_at"].isoformat(),
"last_occurred_at": alert["last_occurred_at"].isoformat(),
"workflow_id": str(workflow_id),
}
field_ids = alert_config.get("field_ids")
if isinstance(field_ids, dict):
for value_name, field_id in field_ids.items():
if value_name in alert_values and isinstance(field_id, str) and field_id:
fields[field_id] = alert_values[value_name]
remote_item_id = alert["remote_item_id"]
if remote_item_id:
await self._command(
workflow_id,
"alert_update",
"crm.item.update",
{
"entityTypeId": int(entity_type_id),
"id": remote_item_id,
"fields": fields,
},
mutating=True,
)
return
result = await self._command(
workflow_id,
"alert_add",
"crm.item.add",
{"entityTypeId": int(entity_type_id), "fields": fields},
mutating=True,
)
payload = result.result if isinstance(result.result, dict) else {}
item = payload.get("item", payload)
remote_item_id = item.get("id") if isinstance(item, dict) else None
if remote_item_id is None:
raise RetryableWorkflow("alert_item_id_missing")
async with self.repository.transaction() as connection:
await connection.execute(
text(
"""
INSERT INTO bitrix_sync.business_alerts
(id,fingerprint,alert_type,severity,app_user_id,candidate_external_ids,
workflow_id,status,occurrence_count,first_occurred_at,last_occurred_at,
created_at,updated_at)
VALUES (gen_random_uuid(),encode(digest(:fingerprint,'sha256'),'hex'),
:type,'warning',:user_id,:candidates,:workflow_id,'open',1,
now(),now(),now(),now())
ON CONFLICT (alert_type,fingerprint) WHERE status='open'
DO UPDATE SET occurrence_count=business_alerts.occurrence_count+1,
last_occurred_at=now(),updated_at=now()
UPDATE bitrix_sync.business_alerts
SET remote_item_id=:remote_item_id,remote_stage_id=:stage_id,updated_at=now()
WHERE id=:id
"""
),
{
"fingerprint": fingerprint,
"type": alert_type,
"user_id": user_id,
"candidates": candidates,
"workflow_id": workflow_id,
"id": alert["id"],
"remote_item_id": str(remote_item_id),
"stage_id": str(stage_new),
},
)
VM2_services/codebase/services/bitrix-sync/app/reconciliation.py
+28 -15
View File
@@ -45,13 +45,13 @@ class IncrementalReconciler:
started_at = datetime.now(UTC)
since = (cursor or datetime(1970, 1, 1, tzinfo=UTC)) - timedelta(seconds=overlap_seconds)
start = 0
scanned: list[str] = []
scanned: list[tuple[str, str]] = []
while True:
result = await self.crm.call(
"crm.item.list",
{
"entityTypeId": 3,
"select": ["id"],
"select": ["id", "updatedTime"],
"filter": {
">=updatedTime": since.replace(tzinfo=None).isoformat(timespec="seconds"),
"opened": 1,
@@ -63,10 +63,15 @@ class IncrementalReconciler:
)
if result.outcome != CrmOutcome.SUCCEEDED:
raise RuntimeError(result.error_code or "reconciliation_failed")
payload: dict[str, Any] = result.result or {}
items = payload.get("items", payload if isinstance(payload, list) else [])
scanned.extend(str(item["id"]) for item in items if "id" in item)
next_start = payload.get("next")
payload: Any = result.result or {}
items = payload.get("items", []) if isinstance(payload, dict) else payload
if not isinstance(items, list):
raise RuntimeError("reconciliation_items_invalid")
for item in items:
if not isinstance(item, dict) or "id" not in item or "updatedTime" not in item:
raise RuntimeError("reconciliation_item_missing_identity")
scanned.append((str(item["id"]), str(item["updatedTime"])))
next_start = payload.get("next") if isinstance(payload, dict) else None
if next_start is None:
break
start = int(next_start)
@@ -89,27 +94,35 @@ class IncrementalReconciler:
)
return len(scanned)
async def _enqueue_changed(self, external_ids: list[str]) -> None:
if not external_ids:
async def _enqueue_changed(self, changed_contacts: list[tuple[str, str]]) -> None:
if not changed_contacts:
return
external_ids = [external_id for external_id, _ in changed_contacts]
event_ids = [
f"contact.reconciliation:{external_id}:{updated_time}"
for external_id, updated_time in changed_contacts
]
async with self.repository.transaction() as connection:
await connection.execute(
text(
"""
INSERT INTO bitrix_sync.webhook_inbox
(id,receiver_type,event_type,external_entity_id,status,coalesced_count,
received_at,last_received_at)
SELECT gen_random_uuid(),'contact','contact.reconciliation',id,'received',1,
now(),now()
FROM unnest(CAST(:ids AS text[])) id
(id,receiver_type,event_type,event_id,external_entity_id,status,
coalesced_count,received_at,last_received_at)
SELECT gen_random_uuid(),'contact','contact.reconciliation',candidate.event_id,
candidate.external_id,'received',1,now(),now()
FROM unnest(CAST(:ids AS text[]),CAST(:event_ids AS text[]))
AS candidate(external_id,event_id)
WHERE NOT EXISTS (
SELECT 1 FROM bitrix_sync.webhook_inbox w
WHERE w.receiver_type='contact' AND w.external_entity_id=id
WHERE w.receiver_type='contact'
AND w.external_entity_id=candidate.external_id
AND w.status IN ('received','processing','retry_wait')
)
ON CONFLICT (receiver_type,event_id) WHERE event_id IS NOT NULL DO NOTHING
"""
),
{"ids": external_ids},
{"ids": external_ids, "event_ids": event_ids},
)
VM2_services/codebase/services/bitrix-sync/app/repository.py
+45 -17
View File
@@ -12,6 +12,8 @@ from typing import Any
from sqlalchemy import text
from sqlalchemy.ext.asyncio import AsyncConnection, AsyncEngine, create_async_engine
from app.domain import safe_hash
def postgres_ssl_context() -> ssl.SSLContext:
ca_file = os.environ.get("PG_CA_FILE")
@@ -83,9 +85,12 @@ class Repository:
WITH candidates AS (
SELECT id
FROM han_app.sync_queue
WHERE status IN ('pending','retry_wait')
AND next_attempt_at <= now()
AND (locked_until IS NULL OR locked_until < now())
WHERE (
status IN ('pending','retry_wait') AND next_attempt_at <= now()
)
OR (
status='leased' AND locked_until < now()
)
ORDER BY next_attempt_at, created_at
FOR UPDATE SKIP LOCKED
LIMIT :limit
@@ -156,7 +161,7 @@ class Repository:
"""
)
async with self.engine.begin() as connection:
return (
workflow_id = (
await connection.execute(
sql,
{
@@ -167,6 +172,17 @@ class Repository:
},
)
).scalar_one()
await connection.execute(
text(
"""
UPDATE bitrix_sync.workflow_instances
SET state='running',current_step='load_profile',updated_at=now()
WHERE id=:id AND state IN ('created','running')
"""
),
{"id": workflow_id},
)
return workflow_id
async def complete_task(self, task: LeasedTask, workflow_id: uuid.UUID) -> bool:
async with self.engine.begin() as connection:
@@ -280,8 +296,12 @@ class Repository:
"""
WITH candidates AS (
SELECT id FROM bitrix_sync.webhook_inbox
WHERE status IN ('received','retry_wait') AND next_attempt_at<=now()
AND (locked_until IS NULL OR locked_until<now())
WHERE (
status IN ('received','retry_wait') AND next_attempt_at<=now()
)
OR (
status='processing' AND locked_until<now()
)
ORDER BY next_attempt_at,received_at
FOR UPDATE SKIP LOCKED LIMIT :limit
)
@@ -345,7 +365,8 @@ class Repository:
text(
"""
UPDATE han_app.client_profiles
SET full_name=:full_name,citizenship=:citizenship,email=:email,updated_at=now()
SET full_name=:full_name,citizenship=:citizenship,email=:email,
source_updated_at=:source_updated_at,updated_at=now()
WHERE user_id=:user_id AND record_status='A'
"""
),
@@ -354,6 +375,7 @@ class Repository:
"full_name": full_name,
"citizenship": citizenship,
"email": email,
"source_updated_at": source_updated_at,
},
)
await connection.execute(
@@ -363,14 +385,20 @@ class Repository:
(id,mapping_id,user_id,external_id,full_name_hash,email_hash,
citizenship_hash,source_updated_at,last_applied_source,
last_webhook_received_at,created_at,updated_at)
SELECT gen_random_uuid(),m.id,:user_id,:external_id,
encode(digest(coalesce(:full_name,''),'sha256'),'hex'),
encode(digest(coalesce(:email,''),'sha256'),'hex'),
encode(digest(coalesce(:citizenship,''),'sha256'),'hex'),
:source_updated_at,:source,
CASE WHEN :source='webhook' THEN now() END,now(),now()
SELECT gen_random_uuid(),m.id,:user_id,
CAST(:external_id AS varchar(128)),
CAST(:full_name_hash AS varchar(64)),
CAST(:email_hash AS varchar(64)),
CAST(:citizenship_hash AS varchar(64)),
CAST(:source_updated_at AS timestamptz),
CAST(:source AS varchar(24)),
CASE WHEN CAST(:source AS varchar(24))='webhook'
THEN now() END,
now(),now()
FROM bitrix_sync.entity_external_mapping m
WHERE m.entity_id=:user_id AND m.external_id=:external_id AND m.status='active'
WHERE m.entity_id=:user_id
AND m.external_id=CAST(:external_id AS varchar(128))
AND m.status='active'
ON CONFLICT (mapping_id) DO UPDATE
SET full_name_hash=excluded.full_name_hash,email_hash=excluded.email_hash,
citizenship_hash=excluded.citizenship_hash,
@@ -385,9 +413,9 @@ class Repository:
{
"user_id": user_id,
"external_id": external_id,
"full_name": full_name,
"citizenship": citizenship,
"email": email,
"full_name_hash": safe_hash(full_name or ""),
"citizenship_hash": safe_hash(citizenship or ""),
"email_hash": safe_hash(email or ""),
"source_updated_at": source_updated_at,
"source": source,
},
VM2_services/codebase/services/bitrix-sync/compose.fragment.yaml
+1
View File
@@ -23,6 +23,7 @@ services:
BITRIX_SYNC_CONTACT_USER_ID_FIELD: ${BITRIX_SYNC_CONTACT_USER_ID_FIELD:-}
BITRIX_SYNC_CONTACT_REGISTERED_FIELD: ${BITRIX_SYNC_CONTACT_REGISTERED_FIELD:-}
BITRIX_SYNC_CONTACT_CITIZENSHIP_FIELD: ${BITRIX_SYNC_CONTACT_CITIZENSHIP_FIELD:-}
BITRIX_SYNC_CONTACT_SOURCE: ${BITRIX_SYNC_CONTACT_SOURCE:-}
BITRIX_SYNC_WEBHOOK_ALLOWED_CIDRS: ${BITRIX_WEBHOOK_ALLOWED_CIDRS:-}
volumes: &sync_secrets
- /run/han-chat/secrets/bitrix-sync/database-url:/run/secrets/bitrix_sync_database_url:ro
VM2_services/codebase/services/bitrix-sync/tests/conftest.py
+1
View File
@@ -21,5 +21,6 @@ def full_settings() -> Settings:
contact_user_id_field="UF_CRM_100",
contact_registered_field="UF_CRM_101",
contact_citizenship_field="UF_CRM_102",
contact_source="WEB",
webhook_allowed_cidrs="203.0.113.0/24",
)
VM2_services/codebase/services/bitrix-sync/tests/test_config.py
+5
View File
@@ -31,6 +31,7 @@ def test_full_mode_rejects_portal_host_mismatch() -> None:
contact_user_id_field="UF_CRM_1",
contact_registered_field="UF_CRM_2",
contact_citizenship_field="UF_CRM_3",
contact_source="WEB",
webhook_allowed_cidrs="203.0.113.0/24",
)
@@ -56,6 +57,10 @@ def test_image_default_allows_compose_process_role_override() -> None:
for source in (compose, production_compose):
assert 'command: ["han-bitrix-sync-worker"]' in source
assert 'command: ["han-bitrix-sync-reconciliation"]' in source
reconciliation_service = source.split(" bitrix-sync-reconciliation:", 1)[1].split(
"\n bitrix-sync-", 1
)[0]
assert 'restart: "no"' in reconciliation_service
def test_alembic_chain_preserves_legacy_baseline() -> None:
VM2_services/codebase/services/bitrix-sync/tests/test_engine_boundaries.py
+110 -3
View File
@@ -2,6 +2,7 @@ from __future__ import annotations
import uuid
from contextlib import asynccontextmanager
from datetime import UTC, datetime
import pytest
from sqlalchemy.dialects.postgresql import JSONB
@@ -14,9 +15,54 @@ from app.repository import Profile
class Result:
rowcount = 1
def __init__(self, *, row=None, scalar=None) -> None:
self.row = row
self.scalar = scalar
def mappings(self):
return self
def one(self):
return self.row
def scalar_one_or_none(self):
return self.scalar
class Connection:
def __init__(self, statements: list[str]) -> None:
self.statements = statements
async def execute(self, statement, params=None):
sql = str(statement)
self.statements.append(sql)
if "INSERT INTO bitrix_sync.business_alerts" in sql:
now = datetime.now(UTC)
return Result(
row={
"id": uuid.uuid4(),
"alert_number": 1,
"fingerprint": "fingerprint",
"alert_type": params["type"],
"severity": "warning",
"app_user_id": params["user_id"],
"selected_external_id": params["selected_external_id"],
"candidate_external_ids": params["candidates"],
"remote_item_id": None,
"occurrence_count": 1,
"first_occurred_at": now,
"last_occurred_at": now,
}
)
if "SELECT value_json FROM bitrix_sync.settings" in sql:
return Result(
scalar={
"entity_type_id": 178,
"category_id": 5,
"stage_new": "DT178_5:NEW",
"field_ids": {"candidate_external_ids": "ufCrm_999"},
}
)
return Result()
@@ -27,7 +73,7 @@ class FakeRepository:
@asynccontextmanager
async def transaction(self):
yield Connection()
yield Connection(self.statements)
async def active_mapping(self, user_id):
return self.mapping
@@ -37,8 +83,16 @@ class FakeRepository:
class FakeCrm:
def __init__(self, user_id: uuid.UUID) -> None:
def __init__(
self,
user_id: uuid.UUID,
*,
existing_identity: bool = False,
foreign_owner: bool = False,
) -> None:
self.user_id = user_id
self.existing_identity = existing_identity
self.foreign_owner = foreign_owner
self.calls: list[tuple[str, dict]] = []
async def call(self, method, params, *, mutating):
@@ -52,9 +106,19 @@ class FakeCrm:
{
"ID": contact_id,
"CREATED_TIME": "2026-08-06T10:00:00Z",
"UF_CRM_100": None,
"UF_CRM_100": (
str(self.user_id)
if self.existing_identity and contact_id == "10"
else str(uuid.UUID(int=1))
if self.foreign_owner and contact_id == "10"
else None
),
},
)
if method == "crm.contact.add":
return CrmResult(CrmOutcome.SUCCEEDED, "11")
if method == "crm.item.add":
return CrmResult(CrmOutcome.SUCCEEDED, {"item": {"id": "500"}})
return CrmResult(CrmOutcome.SUCCEEDED, True)
@@ -76,3 +140,46 @@ async def test_multiple_contacts_choose_numeric_newest(full_settings) -> None:
updates = [params for method, params in crm.calls if method == "crm.contact.update"]
assert updates[0]["id"] == "10"
assert not any(method == "crm.contact.add" for method, _ in crm.calls)
assert any(method == "crm.item.add" for method, _ in crm.calls)
alert_add = next(params for method, params in crm.calls if method == "crm.item.add")
assert alert_add["fields"]["contactIds"] == [9, 10]
assert alert_add["fields"]["sourceId"] == "WEB"
assert "ufCrm_999" not in alert_add["fields"]
assert any("entity_external_mapping" in statement for statement in repository.statements)
assert not any("digest(" in statement for statement in repository.statements)
@pytest.mark.asyncio
async def test_recovered_duplicate_creates_alert_before_mapping(full_settings) -> None:
user_id = uuid.uuid4()
repository = FakeRepository()
crm = FakeCrm(user_id, existing_identity=True)
engine = WorkflowEngine(repository, crm, full_settings)
await engine._map_or_create(
uuid.uuid4(),
Profile(user_id=user_id, phone="+79001234567", identity_status="A", profile_status="A"),
)
methods = [method for method, _ in crm.calls]
assert "crm.item.add" in methods
assert "crm.contact.update" not in methods
assert any("entity_external_mapping" in statement for statement in repository.statements)
@pytest.mark.asyncio
async def test_foreign_owned_contact_alert_includes_new_contact(full_settings) -> None:
user_id = uuid.uuid4()
repository = FakeRepository()
crm = FakeCrm(user_id, foreign_owner=True)
engine = WorkflowEngine(repository, crm, full_settings)
await engine._map_or_create(
uuid.uuid4(),
Profile(user_id=user_id, phone="+79001234567", identity_status="A", profile_status="A"),
)
methods = [method for method, _ in crm.calls]
alert_add = next(params for method, params in crm.calls if method == "crm.item.add")
assert methods.index("crm.contact.add") < methods.index("crm.item.add")
assert alert_add["fields"]["contactIds"] == [9, 10, 11]
VM2_services/codebase/services/bitrix-sync/tests/test_repository.py
+122 -1
View File
@@ -1,11 +1,15 @@
from __future__ import annotations
import uuid
from contextlib import asynccontextmanager
from datetime import UTC, datetime
from types import SimpleNamespace
import pytest
from app.repository import Repository
from app.domain import safe_hash
from app.reconciliation import IncrementalReconciler
from app.repository import LeasedTask, Repository
class FakeResult:
@@ -16,9 +20,15 @@ class FakeResult:
def __iter__(self):
return iter(self.rows)
def mappings(self):
return self
def scalar_one_or_none(self):
return self.scalar
def scalar_one(self):
return self.scalar
class FakeConnection:
async def execute(self, statement):
@@ -38,10 +48,29 @@ class FakeConnection:
class FakeEngine:
def __init__(self) -> None:
self.queries: list[str] = []
@asynccontextmanager
async def connect(self):
yield FakeConnection()
@asynccontextmanager
async def begin(self):
yield ClaimConnection(self.queries)
class ClaimConnection:
def __init__(self, queries: list[str]) -> None:
self.queries = queries
async def execute(self, statement, params):
sql = str(statement)
self.queries.append(sql)
if "INSERT INTO bitrix_sync.workflow_instances" in sql:
return FakeResult(scalar=params["id"])
return FakeResult()
@pytest.mark.asyncio
async def test_status_uses_common_status_alias_for_workflows() -> None:
@@ -55,3 +84,95 @@ async def test_status_uses_common_status_alias_for_workflows() -> None:
assert result["commands"] == {"succeeded": 4}
assert result["webhook_lag_seconds"] == 1.5
assert result["settings_version"] == 1
@pytest.mark.asyncio
async def test_claims_recover_expired_leases() -> None:
repository = object.__new__(Repository)
engine = FakeEngine()
repository.engine = engine
assert await repository.claim_tasks("worker", 10, 60) == []
assert await repository.claim_webhooks("worker", 10, 60) == []
assert "status='leased' AND locked_until < now()" in engine.queries[0]
assert "status='processing' AND locked_until<now()" in engine.queries[1]
@pytest.mark.asyncio
async def test_create_workflow_marks_recovered_workflow_running() -> None:
repository = object.__new__(Repository)
engine = FakeEngine()
repository.engine = engine
task = LeasedTask(uuid.uuid4(), "contact.map_or_create", uuid.uuid4(), uuid.uuid4(), 0)
workflow_id = await repository.create_workflow(task)
assert isinstance(workflow_id, uuid.UUID)
assert "SET state='running'" in engine.queries[1]
@pytest.mark.asyncio
async def test_reconciliation_uses_unambiguous_external_id_alias() -> None:
queries: list[str] = []
class Connection:
async def execute(self, statement, params):
queries.append(str(statement))
return FakeResult()
class ReconciliationRepository:
@asynccontextmanager
async def transaction(self):
yield Connection()
reconciler = IncrementalReconciler(ReconciliationRepository(), None, "ufCrm_1")
await reconciler._enqueue_changed(
[
("29406", "2026-08-20T13:21:00+00:00"),
("29502", "2026-08-20T13:22:00+00:00"),
]
)
assert "AS candidate(external_id,event_id)" in queries[0]
assert "w.external_entity_id=candidate.external_id" in queries[0]
assert "ON CONFLICT (receiver_type,event_id)" in queries[0]
@pytest.mark.asyncio
async def test_apply_crm_profile_hashes_without_database_digest() -> None:
calls: list[tuple[str, dict | None]] = []
class Connection:
async def execute(self, statement, params=None):
calls.append((str(statement), params))
return FakeResult()
class Engine:
@asynccontextmanager
async def begin(self):
yield Connection()
repository = object.__new__(Repository)
repository.engine = Engine()
source_updated_at = datetime.now(UTC)
await repository.apply_crm_profile(
uuid.uuid4(),
"29406",
full_name="Тестовый пользователь",
citizenship=None,
email=None,
source_updated_at=source_updated_at,
source="webhook",
)
profile_sql, profile_params = calls[1]
snapshot_sql, snapshot_params = calls[2]
assert "source_updated_at=:source_updated_at" in profile_sql
assert profile_params["source_updated_at"] == source_updated_at
assert "digest(" not in snapshot_sql
assert "CAST(:external_id AS varchar(128))" in snapshot_sql
assert "CAST(:source AS varchar(24))" in snapshot_sql
assert snapshot_params["full_name_hash"] == safe_hash("Тестовый пользователь")
assert snapshot_params["email_hash"] == safe_hash("")
VM2_services/codebase/services/docker-compose.yml
+2
View File
@@ -44,6 +44,7 @@ x-bitrix-sync-environment: &bitrix-sync-environment
BITRIX_SYNC_CONTACT_USER_ID_FIELD: ${BITRIX_SYNC_CONTACT_USER_ID_FIELD:?set contact field}
BITRIX_SYNC_CONTACT_REGISTERED_FIELD: ${BITRIX_SYNC_CONTACT_REGISTERED_FIELD:?set registration field}
BITRIX_SYNC_CONTACT_CITIZENSHIP_FIELD: ${BITRIX_SYNC_CONTACT_CITIZENSHIP_FIELD:?set citizenship field}
BITRIX_SYNC_CONTACT_SOURCE: ${BITRIX_SYNC_CONTACT_SOURCE:?set contact source}
BITRIX_SYNC_WEBHOOK_ALLOWED_CIDRS: ${BITRIX_WEBHOOK_ALLOWED_CIDRS:-}
BITRIX_SYNC_HTTP_TIMEOUT_SEC: ${BITRIX_SYNC_HTTP_TIMEOUT_SEC:-10}
BITRIX_SYNC_DB_POOL_SIZE: ${BITRIX_SYNC_DB_POOL_SIZE:-5}
@@ -372,6 +373,7 @@ services:
image: ${BITRIX_SYNC_IMAGE:?set immutable bitrix-sync image digest}
command: ["han-bitrix-sync-reconciliation"]
user: "10001:10001"
restart: "no"
environment: *bitrix-sync-environment
volumes:
- *postgres-ca-volume