mi
137 lines
5.0 KiB
Python
137 lines
5.0 KiB
Python
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import signal
|
|
import socket
|
|
import uuid
|
|
from time import monotonic
|
|
|
|
from app.config import load_settings
|
|
from app.crm import CrmClient
|
|
from app.domain import full_jitter_delay
|
|
from app.engine import RetryableWorkflow, WorkflowEngine
|
|
from app.repository import Repository
|
|
from app.telemetry import (
|
|
init_telemetry,
|
|
log_event,
|
|
record_claim,
|
|
record_retry,
|
|
record_workflow,
|
|
safe_span,
|
|
shutdown_telemetry,
|
|
)
|
|
|
|
|
|
async def worker_main() -> None:
|
|
init_telemetry("bitrix-sync-worker")
|
|
repository: Repository | None = None
|
|
crm: CrmClient | None = None
|
|
try:
|
|
settings = load_settings()
|
|
if not settings.enabled:
|
|
log_event("worker.disabled", "Bitrix sync worker is disabled")
|
|
return
|
|
assert settings.database_url and settings.crm_rest_webhook_url and settings.portal_host
|
|
repository = Repository(settings.database_url.get_secret_value(), settings.db_pool_size)
|
|
crm = CrmClient(
|
|
settings.crm_rest_webhook_url.get_secret_value(),
|
|
settings.portal_host,
|
|
settings.http_timeout_sec,
|
|
)
|
|
engine = WorkflowEngine(repository, crm, settings)
|
|
stop = asyncio.Event()
|
|
loop = asyncio.get_running_loop()
|
|
for event in (signal.SIGINT, signal.SIGTERM):
|
|
try:
|
|
loop.add_signal_handler(event, stop.set)
|
|
except NotImplementedError:
|
|
pass
|
|
worker_id = f"{socket.gethostname()}:{uuid.uuid4()}"
|
|
log_event("worker.started", "Bitrix sync worker started")
|
|
while not stop.is_set():
|
|
with safe_span("bitrix_sync.worker.claim"):
|
|
tasks = await repository.claim_tasks(
|
|
worker_id, settings.claim_size, settings.lease_seconds
|
|
)
|
|
webhooks = await repository.claim_webhooks(
|
|
worker_id, settings.claim_size, settings.lease_seconds
|
|
)
|
|
rebind_ids = await repository.pending_rebind_ids(settings.claim_size)
|
|
record_claim("task", len(tasks))
|
|
record_claim("webhook", len(webhooks))
|
|
record_claim("rebind", len(rebind_ids))
|
|
if not tasks and not webhooks and not rebind_ids:
|
|
try:
|
|
await asyncio.wait_for(stop.wait(), timeout=1)
|
|
except TimeoutError:
|
|
continue
|
|
for task in tasks:
|
|
if stop.is_set():
|
|
break
|
|
started = monotonic()
|
|
outcome = "success"
|
|
try:
|
|
with safe_span(
|
|
"bitrix_sync.worker.process",
|
|
attributes={"workflow.type": task.task_type},
|
|
):
|
|
await engine.process(task)
|
|
except Exception:
|
|
outcome = "error"
|
|
raise
|
|
finally:
|
|
record_workflow(task.task_type, outcome, monotonic() - started)
|
|
for item in webhooks:
|
|
if stop.is_set():
|
|
break
|
|
started = monotonic()
|
|
outcome = "success"
|
|
try:
|
|
with safe_span(
|
|
"bitrix_sync.worker.webhook",
|
|
attributes={"receiver": item.receiver_type},
|
|
):
|
|
await engine.process_webhook(item)
|
|
except RetryableWorkflow as exc:
|
|
outcome = "retry"
|
|
record_retry("webhook")
|
|
delay = exc.retry_after or full_jitter_delay(
|
|
item.attempt_count + 1,
|
|
settings.retry_base_seconds,
|
|
settings.retry_max_seconds,
|
|
)
|
|
await repository.retry_webhook(item, exc.code, delay)
|
|
except Exception:
|
|
outcome = "error"
|
|
raise
|
|
finally:
|
|
record_workflow("contact.webhook", outcome, monotonic() - started)
|
|
for request_id in rebind_ids:
|
|
if stop.is_set():
|
|
break
|
|
started = monotonic()
|
|
outcome = "success"
|
|
try:
|
|
with safe_span("bitrix_sync.worker.rebind"):
|
|
await engine.process_rebind(request_id)
|
|
except RetryableWorkflow:
|
|
outcome = "retry"
|
|
record_retry("rebind")
|
|
continue
|
|
except Exception:
|
|
outcome = "error"
|
|
raise
|
|
finally:
|
|
record_workflow("rebind", outcome, monotonic() - started)
|
|
finally:
|
|
if crm is not None:
|
|
await crm.close()
|
|
if repository is not None:
|
|
await repository.close()
|
|
log_event("worker.stopped", "Bitrix sync worker stopped")
|
|
shutdown_telemetry()
|
|
|
|
|
|
def run() -> None:
|
|
asyncio.run(worker_main())
|