Добавлен сбор телеметрии на ВМ2
This commit is contained in:
mi
@@ -4,43 +4,62 @@ import asyncio
|
||||
import signal
|
||||
import socket
|
||||
import uuid
|
||||
from time import monotonic
|
||||
|
||||
from app.config import load_settings
|
||||
from app.crm import CrmClient
|
||||
from app.domain import full_jitter_delay
|
||||
from app.engine import RetryableWorkflow, WorkflowEngine
|
||||
from app.repository import Repository
|
||||
from app.telemetry import (
|
||||
init_telemetry,
|
||||
log_event,
|
||||
record_claim,
|
||||
record_retry,
|
||||
record_workflow,
|
||||
safe_span,
|
||||
shutdown_telemetry,
|
||||
)
|
||||
|
||||
|
||||
async def worker_main() -> None:
|
||||
settings = load_settings()
|
||||
if not settings.enabled:
|
||||
return
|
||||
assert settings.database_url and settings.crm_rest_webhook_url and settings.portal_host
|
||||
repository = Repository(settings.database_url.get_secret_value(), settings.db_pool_size)
|
||||
crm = CrmClient(
|
||||
settings.crm_rest_webhook_url.get_secret_value(),
|
||||
settings.portal_host,
|
||||
settings.http_timeout_sec,
|
||||
)
|
||||
engine = WorkflowEngine(repository, crm, settings)
|
||||
stop = asyncio.Event()
|
||||
loop = asyncio.get_running_loop()
|
||||
for event in (signal.SIGINT, signal.SIGTERM):
|
||||
try:
|
||||
loop.add_signal_handler(event, stop.set)
|
||||
except NotImplementedError:
|
||||
pass
|
||||
worker_id = f"{socket.gethostname()}:{uuid.uuid4()}"
|
||||
init_telemetry("bitrix-sync-worker")
|
||||
repository: Repository | None = None
|
||||
crm: CrmClient | None = None
|
||||
try:
|
||||
settings = load_settings()
|
||||
if not settings.enabled:
|
||||
log_event("worker.disabled", "Bitrix sync worker is disabled")
|
||||
return
|
||||
assert settings.database_url and settings.crm_rest_webhook_url and settings.portal_host
|
||||
repository = Repository(settings.database_url.get_secret_value(), settings.db_pool_size)
|
||||
crm = CrmClient(
|
||||
settings.crm_rest_webhook_url.get_secret_value(),
|
||||
settings.portal_host,
|
||||
settings.http_timeout_sec,
|
||||
)
|
||||
engine = WorkflowEngine(repository, crm, settings)
|
||||
stop = asyncio.Event()
|
||||
loop = asyncio.get_running_loop()
|
||||
for event in (signal.SIGINT, signal.SIGTERM):
|
||||
try:
|
||||
loop.add_signal_handler(event, stop.set)
|
||||
except NotImplementedError:
|
||||
pass
|
||||
worker_id = f"{socket.gethostname()}:{uuid.uuid4()}"
|
||||
log_event("worker.started", "Bitrix sync worker started")
|
||||
while not stop.is_set():
|
||||
tasks = await repository.claim_tasks(
|
||||
worker_id, settings.claim_size, settings.lease_seconds
|
||||
)
|
||||
webhooks = await repository.claim_webhooks(
|
||||
worker_id, settings.claim_size, settings.lease_seconds
|
||||
)
|
||||
rebind_ids = await repository.pending_rebind_ids(settings.claim_size)
|
||||
with safe_span("bitrix_sync.worker.claim"):
|
||||
tasks = await repository.claim_tasks(
|
||||
worker_id, settings.claim_size, settings.lease_seconds
|
||||
)
|
||||
webhooks = await repository.claim_webhooks(
|
||||
worker_id, settings.claim_size, settings.lease_seconds
|
||||
)
|
||||
rebind_ids = await repository.pending_rebind_ids(settings.claim_size)
|
||||
record_claim("task", len(tasks))
|
||||
record_claim("webhook", len(webhooks))
|
||||
record_claim("rebind", len(rebind_ids))
|
||||
if not tasks and not webhooks and not rebind_ids:
|
||||
try:
|
||||
await asyncio.wait_for(stop.wait(), timeout=1)
|
||||
@@ -49,29 +68,68 @@ async def worker_main() -> None:
|
||||
for task in tasks:
|
||||
if stop.is_set():
|
||||
break
|
||||
await engine.process(task)
|
||||
started = monotonic()
|
||||
outcome = "success"
|
||||
try:
|
||||
with safe_span(
|
||||
"bitrix_sync.worker.process",
|
||||
attributes={"workflow.type": task.task_type},
|
||||
):
|
||||
await engine.process(task)
|
||||
except Exception:
|
||||
outcome = "error"
|
||||
raise
|
||||
finally:
|
||||
record_workflow(task.task_type, outcome, monotonic() - started)
|
||||
for item in webhooks:
|
||||
if stop.is_set():
|
||||
break
|
||||
started = monotonic()
|
||||
outcome = "success"
|
||||
try:
|
||||
await engine.process_webhook(item)
|
||||
with safe_span(
|
||||
"bitrix_sync.worker.webhook",
|
||||
attributes={"receiver": item.receiver_type},
|
||||
):
|
||||
await engine.process_webhook(item)
|
||||
except RetryableWorkflow as exc:
|
||||
outcome = "retry"
|
||||
record_retry("webhook")
|
||||
delay = exc.retry_after or full_jitter_delay(
|
||||
item.attempt_count + 1,
|
||||
settings.retry_base_seconds,
|
||||
settings.retry_max_seconds,
|
||||
)
|
||||
await repository.retry_webhook(item, exc.code, delay)
|
||||
except Exception:
|
||||
outcome = "error"
|
||||
raise
|
||||
finally:
|
||||
record_workflow("contact.webhook", outcome, monotonic() - started)
|
||||
for request_id in rebind_ids:
|
||||
if stop.is_set():
|
||||
break
|
||||
started = monotonic()
|
||||
outcome = "success"
|
||||
try:
|
||||
await engine.process_rebind(request_id)
|
||||
with safe_span("bitrix_sync.worker.rebind"):
|
||||
await engine.process_rebind(request_id)
|
||||
except RetryableWorkflow:
|
||||
outcome = "retry"
|
||||
record_retry("rebind")
|
||||
continue
|
||||
except Exception:
|
||||
outcome = "error"
|
||||
raise
|
||||
finally:
|
||||
record_workflow("rebind", outcome, monotonic() - started)
|
||||
finally:
|
||||
await crm.close()
|
||||
await repository.close()
|
||||
if crm is not None:
|
||||
await crm.close()
|
||||
if repository is not None:
|
||||
await repository.close()
|
||||
log_event("worker.stopped", "Bitrix sync worker stopped")
|
||||
shutdown_telemetry()
|
||||
|
||||
|
||||
def run() -> None:
|
||||
|
||||
Reference in New Issue
Block a user