from __future__ import annotations import asyncio import signal import socket import uuid from time import monotonic from app.config import load_settings from app.crm import CrmClient from app.domain import full_jitter_delay from app.engine import RetryableWorkflow, WorkflowEngine from app.repository import Repository from app.telemetry import ( init_telemetry, log_event, record_claim, record_retry, record_workflow, safe_span, shutdown_telemetry, ) async def worker_main() -> None: init_telemetry("bitrix-sync-worker") repository: Repository | None = None crm: CrmClient | None = None try: settings = load_settings() if not settings.enabled: log_event("worker.disabled", "Bitrix sync worker is disabled") return assert settings.database_url and settings.crm_rest_webhook_url and settings.portal_host repository = Repository(settings.database_url.get_secret_value(), settings.db_pool_size) crm = CrmClient( settings.crm_rest_webhook_url.get_secret_value(), settings.portal_host, settings.http_timeout_sec, ) engine = WorkflowEngine(repository, crm, settings) stop = asyncio.Event() loop = asyncio.get_running_loop() for event in (signal.SIGINT, signal.SIGTERM): try: loop.add_signal_handler(event, stop.set) except NotImplementedError: pass worker_id = f"{socket.gethostname()}:{uuid.uuid4()}" log_event("worker.started", "Bitrix sync worker started") while not stop.is_set(): with safe_span("bitrix_sync.worker.claim"): tasks = await repository.claim_tasks( worker_id, settings.claim_size, settings.lease_seconds ) webhooks = await repository.claim_webhooks( worker_id, settings.claim_size, settings.lease_seconds ) rebind_ids = await repository.pending_rebind_ids(settings.claim_size) record_claim("task", len(tasks)) record_claim("webhook", len(webhooks)) record_claim("rebind", len(rebind_ids)) if not tasks and not webhooks and not rebind_ids: try: await asyncio.wait_for(stop.wait(), timeout=1) except TimeoutError: continue for task in tasks: if stop.is_set(): break started = monotonic() outcome = "success" try: with safe_span( "bitrix_sync.worker.process", attributes={"workflow.type": task.task_type}, ): await engine.process(task) except Exception: outcome = "error" raise finally: record_workflow(task.task_type, outcome, monotonic() - started) for item in webhooks: if stop.is_set(): break started = monotonic() outcome = "success" try: with safe_span( "bitrix_sync.worker.webhook", attributes={"receiver": item.receiver_type}, ): await engine.process_webhook(item) except RetryableWorkflow as exc: outcome = "retry" record_retry("webhook") delay = exc.retry_after or full_jitter_delay( item.attempt_count + 1, settings.retry_base_seconds, settings.retry_max_seconds, ) await repository.retry_webhook(item, exc.code, delay) except Exception: outcome = "error" raise finally: record_workflow("contact.webhook", outcome, monotonic() - started) for request_id in rebind_ids: if stop.is_set(): break started = monotonic() outcome = "success" try: with safe_span("bitrix_sync.worker.rebind"): await engine.process_rebind(request_id) except RetryableWorkflow: outcome = "retry" record_retry("rebind") continue except Exception: outcome = "error" raise finally: record_workflow("rebind", outcome, monotonic() - started) finally: if crm is not None: await crm.close() if repository is not None: await repository.close() log_event("worker.stopped", "Bitrix sync worker stopped") shutdown_telemetry() def run() -> None: asyncio.run(worker_main())