from __future__ import annotations import asyncio from datetime import UTC, datetime, timedelta from typing import Any from sqlalchemy import text from app.config import load_settings from app.crm import CrmClient, CrmOutcome from app.repository import Repository class IncrementalReconciler: def __init__(self, repository: Repository, crm: CrmClient, registered_rest_field: str) -> None: self.repository = repository self.crm = crm self.registered_rest_field = registered_rest_field async def run_once(self, overlap_seconds: int) -> int: async with self.repository.transaction() as connection: acquired = ( await connection.execute( text( """ SELECT pg_try_advisory_xact_lock( hashtext('bitrix-contact-reconciliation') ) """ ) ) ).scalar_one() if not acquired: return 0 cursor = ( await connection.execute( text( """ SELECT watermark FROM bitrix_sync.reconciliation_cursors WHERE job_type='contact_incremental' FOR UPDATE """ ) ) ).scalar_one_or_none() started_at = datetime.now(UTC) since = (cursor or datetime(1970, 1, 1, tzinfo=UTC)) - timedelta(seconds=overlap_seconds) start = 0 scanned: list[str] = [] while True: result = await self.crm.call( "crm.item.list", { "entityTypeId": 3, "select": ["id"], "filter": { ">=updatedTime": since.replace(tzinfo=None).isoformat(timespec="seconds"), "opened": 1, self.registered_rest_field: 1, }, "start": start, }, mutating=False, ) if result.outcome != CrmOutcome.SUCCEEDED: raise RuntimeError(result.error_code or "reconciliation_failed") payload: dict[str, Any] = result.result or {} items = payload.get("items", payload if isinstance(payload, list) else []) scanned.extend(str(item["id"]) for item in items if "id" in item) next_start = payload.get("next") if next_start is None: break start = int(next_start) await self._enqueue_changed(scanned) async with self.repository.transaction() as connection: await connection.execute( text( """ INSERT INTO bitrix_sync.reconciliation_cursors (job_type,watermark,overlap_seconds,last_success_at,last_scanned_count, created_at,updated_at) VALUES ('contact_incremental',:watermark,:overlap,now(),:count,now(),now()) ON CONFLICT (job_type) DO UPDATE SET watermark=excluded.watermark,overlap_seconds=excluded.overlap_seconds, last_success_at=now(),last_scanned_count=excluded.last_scanned_count, updated_at=now() """ ), {"watermark": started_at, "overlap": overlap_seconds, "count": len(scanned)}, ) return len(scanned) async def _enqueue_changed(self, external_ids: list[str]) -> None: if not external_ids: return async with self.repository.transaction() as connection: await connection.execute( text( """ INSERT INTO bitrix_sync.webhook_inbox (id,receiver_type,event_type,external_entity_id,status,coalesced_count, received_at,last_received_at) SELECT gen_random_uuid(),'contact','contact.reconciliation',id,'received',1, now(),now() FROM unnest(CAST(:ids AS text[])) id WHERE NOT EXISTS ( SELECT 1 FROM bitrix_sync.webhook_inbox w WHERE w.receiver_type='contact' AND w.external_entity_id=id AND w.status IN ('received','processing','retry_wait') ) """ ), {"ids": external_ids}, ) async def reconciliation_main() -> None: settings = load_settings() if not settings.enabled: return assert settings.database_url and settings.crm_rest_webhook_url and settings.portal_host assert settings.contact_registered_field repository = Repository(settings.database_url.get_secret_value(), settings.db_pool_size) crm = CrmClient( settings.crm_rest_webhook_url.get_secret_value(), settings.portal_host, settings.http_timeout_sec, ) reconciler = IncrementalReconciler( repository, crm, settings.rest_field_name(settings.contact_registered_field) ) try: await reconciler.run_once(settings.reconciliation_overlap_seconds) finally: await crm.close() await repository.close() def run() -> None: asyncio.run(reconciliation_main())