Admin/han-app
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Финальная стабильная реализация

Browse Source
This commit is contained in:
mi
2026-08-21 11:48:24 +03:00
parent 6e24278b10
commit d2415fcfeb
17 changed files with 662 additions and 57 deletions
Download Patch File Download Diff File Expand all files Collapse all files
VM2_services/codebase/services/bitrix-sync/app/config.py
+6
View File
@@ -46,6 +46,7 @@ class Settings(BaseSettings):
contact_user_id_field: str | None = None
contact_registered_field: str | None = None
contact_citizenship_field: str | None = None
contact_source: str | None = Field(default=None, min_length=1, max_length=128)
webhook_allowed_cidrs: str = ""
http_timeout_sec: float = Field(default=10, ge=1, le=60)
@@ -97,6 +98,7 @@ class Settings(BaseSettings):
"contact_user_id_field": self.contact_user_id_field,
"contact_registered_field": self.contact_registered_field,
"contact_citizenship_field": self.contact_citizenship_field,
"contact_source": self.contact_source,
}
missing = [name for name, value in required.items() if not value]
if missing:
@@ -110,6 +112,10 @@ class Settings(BaseSettings):
raise ValueError(f"{name} must match UF_CRM_<latin letters or digits>")
if not MEMBER_RE.fullmatch(str(self.portal_member_id)):
raise ValueError("portal_member_id has invalid format")
assert self.contact_source
self.contact_source = self.contact_source.strip()
if not self.contact_source:
raise ValueError("contact_source cannot be blank")
crm = urlsplit(self.crm_rest_webhook_url.get_secret_value())
public = urlsplit(str(self.public_base_url))
VM2_services/codebase/services/bitrix-sync/app/engine.py
+155 -21
View File
@@ -15,6 +15,7 @@ from app.domain import (
choose_newest,
full_jitter_delay,
parse_crm_datetime,
safe_hash,
select_email,
validate_phone,
)
@@ -199,6 +200,14 @@ class WorkflowEngine:
alert_code: str | None = None
if same_user:
selected_id = str(same_user["ID"])
if len(contacts) > 1:
await self._alert(
workflow_id,
profile.user_id,
"duplicate_contacts",
[str(item["ID"]) for item in contacts],
selected_external_id=selected_id,
)
elif not contacts:
selected_id = await self._create_contact(workflow_id, profile)
else:
@@ -214,15 +223,27 @@ class WorkflowEngine:
assert selected is not None
all_ids = [item.b24_id for item in candidates]
if selected.crm_user_id and selected.crm_user_id != str(profile.user_id):
selected_id = await self._create_contact(workflow_id, profile)
alert_code = "contact_owned_by_other_user"
selected_id = await self._create_contact(workflow_id, profile)
await self._alert(
workflow_id,
profile.user_id,
alert_code,
[*all_ids, selected_id],
selected_external_id=selected_id,
)
else:
selected_id = selected.b24_id
await self._write_identity(workflow_id, selected_id, profile.user_id, active=True)
if len(candidates) > 1:
alert_code = "duplicate_contacts"
if alert_code:
await self._alert(workflow_id, profile.user_id, alert_code, all_ids)
await self._alert(
workflow_id,
profile.user_id,
alert_code,
all_ids,
selected_external_id=selected_id,
)
await self._write_identity(workflow_id, selected_id, profile.user_id, active=True)
await self._activate_mapping(workflow_id, profile.user_id, selected_id)
async def _create_contact(self, workflow_id: uuid.UUID, profile: Profile) -> str:
@@ -551,31 +572,144 @@ class WorkflowEngine:
)
async def _alert(
self, workflow_id: uuid.UUID, user_id: uuid.UUID, alert_type: str, candidates: list[str]
self,
workflow_id: uuid.UUID,
user_id: uuid.UUID,
alert_type: str,
candidates: list[str],
*,
selected_external_id: str | None = None,
) -> None:
fingerprint = f"{alert_type}:{user_id}"
fingerprint = safe_hash(f"{alert_type}:{user_id}")
assert fingerprint is not None
async with self.repository.transaction() as connection:
alert = (
await connection.execute(
text(
"""
INSERT INTO bitrix_sync.business_alerts
(id,fingerprint,alert_type,severity,app_user_id,
selected_external_id,candidate_external_ids,workflow_id,status,
occurrence_count,first_occurred_at,last_occurred_at,created_at,updated_at)
VALUES (gen_random_uuid(),:fingerprint,
:type,'warning',:user_id,:selected_external_id,:candidates,
:workflow_id,'open',1,now(),now(),now(),now())
ON CONFLICT (alert_type,fingerprint) WHERE status='open'
DO UPDATE SET
selected_external_id=coalesce(
excluded.selected_external_id,
business_alerts.selected_external_id
),
candidate_external_ids=excluded.candidate_external_ids,
workflow_id=excluded.workflow_id,
occurrence_count=business_alerts.occurrence_count+1,
last_occurred_at=now(),updated_at=now()
RETURNING id,alert_number,fingerprint,alert_type,severity,app_user_id,
selected_external_id,candidate_external_ids,remote_item_id,
occurrence_count,first_occurred_at,last_occurred_at
"""
),
{
"fingerprint": fingerprint,
"type": alert_type,
"user_id": user_id,
"selected_external_id": selected_external_id,
"candidates": candidates,
"workflow_id": workflow_id,
},
)
).mappings().one()
alert_config = (
await connection.execute(
text(
"""
SELECT value_json FROM bitrix_sync.settings
WHERE key='business_alerts' AND active=true
AND validation_status='valid'
"""
)
)
).scalar_one_or_none()
if not isinstance(alert_config, dict):
raise RetryableWorkflow("business_alerts_not_configured")
entity_type_id = alert_config.get("entity_type_id")
stage_new = alert_config.get("stage_new")
if not entity_type_id or not stage_new:
raise RetryableWorkflow("business_alerts_not_configured")
title = (
f"Конфликт синхронизации #{alert['alert_number']}: {alert_type}; "
f"user={user_id}; contacts={','.join(candidates)}"
)
fields: dict[str, Any] = {
"title": title[:255],
"stageId": stage_new,
"contactIds": [int(candidate) for candidate in candidates if candidate.isdigit()],
"sourceId": self.settings.contact_source,
}
if category_id := alert_config.get("category_id"):
fields["categoryId"] = category_id
if responsible_id := alert_config.get("responsible_id"):
fields["assignedById"] = responsible_id
alert_values = {
"alert_number": str(alert["alert_number"]),
"fingerprint": alert["fingerprint"],
"alert_type": alert["alert_type"],
"severity": alert["severity"],
"app_user_id": str(alert["app_user_id"]),
"selected_external_id": alert["selected_external_id"] or "",
"occurrence_count": alert["occurrence_count"],
"first_occurred_at": alert["first_occurred_at"].isoformat(),
"last_occurred_at": alert["last_occurred_at"].isoformat(),
"workflow_id": str(workflow_id),
}
field_ids = alert_config.get("field_ids")
if isinstance(field_ids, dict):
for value_name, field_id in field_ids.items():
if value_name in alert_values and isinstance(field_id, str) and field_id:
fields[field_id] = alert_values[value_name]
remote_item_id = alert["remote_item_id"]
if remote_item_id:
await self._command(
workflow_id,
"alert_update",
"crm.item.update",
{
"entityTypeId": int(entity_type_id),
"id": remote_item_id,
"fields": fields,
},
mutating=True,
)
return
result = await self._command(
workflow_id,
"alert_add",
"crm.item.add",
{"entityTypeId": int(entity_type_id), "fields": fields},
mutating=True,
)
payload = result.result if isinstance(result.result, dict) else {}
item = payload.get("item", payload)
remote_item_id = item.get("id") if isinstance(item, dict) else None
if remote_item_id is None:
raise RetryableWorkflow("alert_item_id_missing")
async with self.repository.transaction() as connection:
await connection.execute(
text(
"""
INSERT INTO bitrix_sync.business_alerts
(id,fingerprint,alert_type,severity,app_user_id,candidate_external_ids,
workflow_id,status,occurrence_count,first_occurred_at,last_occurred_at,
created_at,updated_at)
VALUES (gen_random_uuid(),encode(digest(:fingerprint,'sha256'),'hex'),
:type,'warning',:user_id,:candidates,:workflow_id,'open',1,
now(),now(),now(),now())
ON CONFLICT (alert_type,fingerprint) WHERE status='open'
DO UPDATE SET occurrence_count=business_alerts.occurrence_count+1,
last_occurred_at=now(),updated_at=now()
UPDATE bitrix_sync.business_alerts
SET remote_item_id=:remote_item_id,remote_stage_id=:stage_id,updated_at=now()
WHERE id=:id
"""
),
{
"fingerprint": fingerprint,
"type": alert_type,
"user_id": user_id,
"candidates": candidates,
"workflow_id": workflow_id,
"id": alert["id"],
"remote_item_id": str(remote_item_id),
"stage_id": str(stage_new),
},
)
VM2_services/codebase/services/bitrix-sync/app/reconciliation.py
+28 -15
View File
@@ -45,13 +45,13 @@ class IncrementalReconciler:
started_at = datetime.now(UTC)
since = (cursor or datetime(1970, 1, 1, tzinfo=UTC)) - timedelta(seconds=overlap_seconds)
start = 0
scanned: list[str] = []
scanned: list[tuple[str, str]] = []
while True:
result = await self.crm.call(
"crm.item.list",
{
"entityTypeId": 3,
"select": ["id"],
"select": ["id", "updatedTime"],
"filter": {
">=updatedTime": since.replace(tzinfo=None).isoformat(timespec="seconds"),
"opened": 1,
@@ -63,10 +63,15 @@ class IncrementalReconciler:
)
if result.outcome != CrmOutcome.SUCCEEDED:
raise RuntimeError(result.error_code or "reconciliation_failed")
payload: dict[str, Any] = result.result or {}
items = payload.get("items", payload if isinstance(payload, list) else [])
scanned.extend(str(item["id"]) for item in items if "id" in item)
next_start = payload.get("next")
payload: Any = result.result or {}
items = payload.get("items", []) if isinstance(payload, dict) else payload
if not isinstance(items, list):
raise RuntimeError("reconciliation_items_invalid")
for item in items:
if not isinstance(item, dict) or "id" not in item or "updatedTime" not in item:
raise RuntimeError("reconciliation_item_missing_identity")
scanned.append((str(item["id"]), str(item["updatedTime"])))
next_start = payload.get("next") if isinstance(payload, dict) else None
if next_start is None:
break
start = int(next_start)
@@ -89,27 +94,35 @@ class IncrementalReconciler:
)
return len(scanned)
async def _enqueue_changed(self, external_ids: list[str]) -> None:
if not external_ids:
async def _enqueue_changed(self, changed_contacts: list[tuple[str, str]]) -> None:
if not changed_contacts:
return
external_ids = [external_id for external_id, _ in changed_contacts]
event_ids = [
f"contact.reconciliation:{external_id}:{updated_time}"
for external_id, updated_time in changed_contacts
]
async with self.repository.transaction() as connection:
await connection.execute(
text(
"""
INSERT INTO bitrix_sync.webhook_inbox
(id,receiver_type,event_type,external_entity_id,status,coalesced_count,
received_at,last_received_at)
SELECT gen_random_uuid(),'contact','contact.reconciliation',id,'received',1,
now(),now()
FROM unnest(CAST(:ids AS text[])) id
(id,receiver_type,event_type,event_id,external_entity_id,status,
coalesced_count,received_at,last_received_at)
SELECT gen_random_uuid(),'contact','contact.reconciliation',candidate.event_id,
candidate.external_id,'received',1,now(),now()
FROM unnest(CAST(:ids AS text[]),CAST(:event_ids AS text[]))
AS candidate(external_id,event_id)
WHERE NOT EXISTS (
SELECT 1 FROM bitrix_sync.webhook_inbox w
WHERE w.receiver_type='contact' AND w.external_entity_id=id
WHERE w.receiver_type='contact'
AND w.external_entity_id=candidate.external_id
AND w.status IN ('received','processing','retry_wait')
)
ON CONFLICT (receiver_type,event_id) WHERE event_id IS NOT NULL DO NOTHING
"""
),
{"ids": external_ids},
{"ids": external_ids, "event_ids": event_ids},
)
VM2_services/codebase/services/bitrix-sync/app/repository.py
+45 -17
View File
@@ -12,6 +12,8 @@ from typing import Any
from sqlalchemy import text
from sqlalchemy.ext.asyncio import AsyncConnection, AsyncEngine, create_async_engine
from app.domain import safe_hash
def postgres_ssl_context() -> ssl.SSLContext:
ca_file = os.environ.get("PG_CA_FILE")
@@ -83,9 +85,12 @@ class Repository:
WITH candidates AS (
SELECT id
FROM han_app.sync_queue
WHERE status IN ('pending','retry_wait')
AND next_attempt_at <= now()
AND (locked_until IS NULL OR locked_until < now())
WHERE (
status IN ('pending','retry_wait') AND next_attempt_at <= now()
)
OR (
status='leased' AND locked_until < now()
)
ORDER BY next_attempt_at, created_at
FOR UPDATE SKIP LOCKED
LIMIT :limit
@@ -156,7 +161,7 @@ class Repository:
"""
)
async with self.engine.begin() as connection:
return (
workflow_id = (
await connection.execute(
sql,
{
@@ -167,6 +172,17 @@ class Repository:
},
)
).scalar_one()
await connection.execute(
text(
"""
UPDATE bitrix_sync.workflow_instances
SET state='running',current_step='load_profile',updated_at=now()
WHERE id=:id AND state IN ('created','running')
"""
),
{"id": workflow_id},
)
return workflow_id
async def complete_task(self, task: LeasedTask, workflow_id: uuid.UUID) -> bool:
async with self.engine.begin() as connection:
@@ -280,8 +296,12 @@ class Repository:
"""
WITH candidates AS (
SELECT id FROM bitrix_sync.webhook_inbox
WHERE status IN ('received','retry_wait') AND next_attempt_at<=now()
AND (locked_until IS NULL OR locked_until<now())
WHERE (
status IN ('received','retry_wait') AND next_attempt_at<=now()
)
OR (
status='processing' AND locked_until<now()
)
ORDER BY next_attempt_at,received_at
FOR UPDATE SKIP LOCKED LIMIT :limit
)
@@ -345,7 +365,8 @@ class Repository:
text(
"""
UPDATE han_app.client_profiles
SET full_name=:full_name,citizenship=:citizenship,email=:email,updated_at=now()
SET full_name=:full_name,citizenship=:citizenship,email=:email,
source_updated_at=:source_updated_at,updated_at=now()
WHERE user_id=:user_id AND record_status='A'
"""
),
@@ -354,6 +375,7 @@ class Repository:
"full_name": full_name,
"citizenship": citizenship,
"email": email,
"source_updated_at": source_updated_at,
},
)
await connection.execute(
@@ -363,14 +385,20 @@ class Repository:
(id,mapping_id,user_id,external_id,full_name_hash,email_hash,
citizenship_hash,source_updated_at,last_applied_source,
last_webhook_received_at,created_at,updated_at)
SELECT gen_random_uuid(),m.id,:user_id,:external_id,
encode(digest(coalesce(:full_name,''),'sha256'),'hex'),
encode(digest(coalesce(:email,''),'sha256'),'hex'),
encode(digest(coalesce(:citizenship,''),'sha256'),'hex'),
:source_updated_at,:source,
CASE WHEN :source='webhook' THEN now() END,now(),now()
SELECT gen_random_uuid(),m.id,:user_id,
CAST(:external_id AS varchar(128)),
CAST(:full_name_hash AS varchar(64)),
CAST(:email_hash AS varchar(64)),
CAST(:citizenship_hash AS varchar(64)),
CAST(:source_updated_at AS timestamptz),
CAST(:source AS varchar(24)),
CASE WHEN CAST(:source AS varchar(24))='webhook'
THEN now() END,
now(),now()
FROM bitrix_sync.entity_external_mapping m
WHERE m.entity_id=:user_id AND m.external_id=:external_id AND m.status='active'
WHERE m.entity_id=:user_id
AND m.external_id=CAST(:external_id AS varchar(128))
AND m.status='active'
ON CONFLICT (mapping_id) DO UPDATE
SET full_name_hash=excluded.full_name_hash,email_hash=excluded.email_hash,
citizenship_hash=excluded.citizenship_hash,
@@ -385,9 +413,9 @@ class Repository:
{
"user_id": user_id,
"external_id": external_id,
"full_name": full_name,
"citizenship": citizenship,
"email": email,
"full_name_hash": safe_hash(full_name or ""),
"citizenship_hash": safe_hash(citizenship or ""),
"email_hash": safe_hash(email or ""),
"source_updated_at": source_updated_at,
"source": source,
},