Финальная стабильная реализация

VM2_services/codebase/services/bitrix-sync/app/reconciliation.py

@@ -45,13 +45,13 @@ class IncrementalReconciler:
         started_at = datetime.now(UTC)
         since = (cursor or datetime(1970, 1, 1, tzinfo=UTC)) - timedelta(seconds=overlap_seconds)
         start = 0
-        scanned: list[str] = []
+        scanned: list[tuple[str, str]] = []
         while True:
             result = await self.crm.call(
                 "crm.item.list",
                 {
                     "entityTypeId": 3,
-                    "select": ["id"],
+                    "select": ["id", "updatedTime"],
                     "filter": {
                         ">=updatedTime": since.replace(tzinfo=None).isoformat(timespec="seconds"),
                         "opened": 1,
@@ -63,10 +63,15 @@ class IncrementalReconciler:
             )
             if result.outcome != CrmOutcome.SUCCEEDED:
                 raise RuntimeError(result.error_code or "reconciliation_failed")
-            payload: dict[str, Any] = result.result or {}
-            items = payload.get("items", payload if isinstance(payload, list) else [])
-            scanned.extend(str(item["id"]) for item in items if "id" in item)
-            next_start = payload.get("next")
+            payload: Any = result.result or {}
+            items = payload.get("items", []) if isinstance(payload, dict) else payload
+            if not isinstance(items, list):
+                raise RuntimeError("reconciliation_items_invalid")
+            for item in items:
+                if not isinstance(item, dict) or "id" not in item or "updatedTime" not in item:
+                    raise RuntimeError("reconciliation_item_missing_identity")
+                scanned.append((str(item["id"]), str(item["updatedTime"])))
+            next_start = payload.get("next") if isinstance(payload, dict) else None
             if next_start is None:
                 break
             start = int(next_start)
@@ -89,27 +94,35 @@ class IncrementalReconciler:
             )
         return len(scanned)
 
-    async def _enqueue_changed(self, external_ids: list[str]) -> None:
-        if not external_ids:
+    async def _enqueue_changed(self, changed_contacts: list[tuple[str, str]]) -> None:
+        if not changed_contacts:
             return
+        external_ids = [external_id for external_id, _ in changed_contacts]
+        event_ids = [
+            f"contact.reconciliation:{external_id}:{updated_time}"
+            for external_id, updated_time in changed_contacts
+        ]
         async with self.repository.transaction() as connection:
             await connection.execute(
                 text(
                     """
                     INSERT INTO bitrix_sync.webhook_inbox
-                      (id,receiver_type,event_type,external_entity_id,status,coalesced_count,
-                       received_at,last_received_at)
-                    SELECT gen_random_uuid(),'contact','contact.reconciliation',id,'received',1,
-                           now(),now()
-                    FROM unnest(CAST(:ids AS text[])) id
+                      (id,receiver_type,event_type,event_id,external_entity_id,status,
+                       coalesced_count,received_at,last_received_at)
+                    SELECT gen_random_uuid(),'contact','contact.reconciliation',candidate.event_id,
+                           candidate.external_id,'received',1,now(),now()
+                    FROM unnest(CAST(:ids AS text[]),CAST(:event_ids AS text[]))
+                         AS candidate(external_id,event_id)
                     WHERE NOT EXISTS (
                       SELECT 1 FROM bitrix_sync.webhook_inbox w
-                      WHERE w.receiver_type='contact' AND w.external_entity_id=id
+                      WHERE w.receiver_type='contact'
+                        AND w.external_entity_id=candidate.external_id
                         AND w.status IN ('received','processing','retry_wait')
                     )
+                    ON CONFLICT (receiver_type,event_id) WHERE event_id IS NOT NULL DO NOTHING
                     """
                 ),
-                {"ids": external_ids},
+                {"ids": external_ids, "event_ids": event_ids},
             )