mi
320 lines
11 KiB
Python
320 lines
11 KiB
Python
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import hmac
|
|
import random
|
|
import time
|
|
import uuid
|
|
from contextlib import asynccontextmanager, suppress
|
|
from dataclasses import dataclass
|
|
from datetime import UTC, datetime, timedelta
|
|
from typing import Annotated, Protocol
|
|
|
|
import uvicorn
|
|
from fastapi import FastAPI, Header, HTTPException, Request
|
|
from fastapi.responses import JSONResponse
|
|
from pydantic import Field, model_validator
|
|
from pydantic_settings import BaseSettings, SettingsConfigDict
|
|
from sqlalchemy import text
|
|
from sqlalchemy.ext.asyncio import AsyncEngine, create_async_engine
|
|
|
|
|
|
def utcnow() -> datetime:
|
|
return datetime.now(UTC)
|
|
|
|
|
|
class Settings(BaseSettings):
|
|
model_config = SettingsConfigDict(extra="ignore")
|
|
app_env: str = "production-like"
|
|
bitrix_sync_enabled: bool = True
|
|
bitrix_sync_database_url: str | None = None
|
|
bitrix_sync_service_token: str = Field(min_length=16)
|
|
bitrix_sync_db_check_interval_sec: float = Field(default=60, ge=0.05)
|
|
bitrix_sync_db_check_timeout_sec: float = Field(default=5, ge=0.05)
|
|
bitrix_sync_db_check_jitter_ratio: float = Field(default=0.1, ge=0, le=0.5)
|
|
bitrix_sync_db_retry_base_sec: float = Field(default=5, ge=0.05)
|
|
bitrix_sync_db_retry_max_sec: float = Field(default=60, ge=0.05)
|
|
bitrix_sync_ready_max_staleness_sec: float = Field(default=150, ge=1)
|
|
bitrix_sync_db_pool_size: int = Field(default=2, ge=1, le=5)
|
|
bitrix_sync_db_pool_recycle_sec: int = Field(default=300, ge=30)
|
|
|
|
@model_validator(mode="after")
|
|
def validate_enabled(self) -> Settings:
|
|
if self.bitrix_sync_enabled and not self.bitrix_sync_database_url:
|
|
raise ValueError("BITRIX_SYNC_DATABASE_URL is required when sync is enabled")
|
|
return self
|
|
|
|
|
|
@dataclass
|
|
class Snapshot:
|
|
state: str
|
|
started_at: datetime
|
|
last_started_at: datetime | None = None
|
|
last_finished_at: datetime | None = None
|
|
last_success_at: datetime | None = None
|
|
success: bool | None = None
|
|
duration_ms: int | None = None
|
|
error_code: str | None = None
|
|
consecutive_failures: int = 0
|
|
next_check_at: datetime | None = None
|
|
worker_running: bool = False
|
|
|
|
|
|
class Probe(Protocol):
|
|
async def check(self) -> None: ...
|
|
async def close(self) -> None: ...
|
|
|
|
|
|
class DatabaseProbe:
|
|
def __init__(self, engine: AsyncEngine) -> None:
|
|
self.engine = engine
|
|
|
|
async def check(self) -> None:
|
|
async with self.engine.connect() as connection:
|
|
result = await connection.scalar(text("SELECT 1"))
|
|
if result != 1:
|
|
raise RuntimeError("unexpected_result")
|
|
|
|
async def close(self) -> None:
|
|
await self.engine.dispose()
|
|
|
|
|
|
def database_url(value: str) -> str:
|
|
if value.startswith("postgresql://"):
|
|
return value.replace("postgresql://", "postgresql+asyncpg://", 1)
|
|
return value
|
|
|
|
|
|
def classify_error(exc: Exception) -> str:
|
|
name = type(exc).__name__.lower()
|
|
text_value = str(exc).lower()
|
|
if isinstance(exc, TimeoutError):
|
|
return "db_query_timeout"
|
|
if "auth" in name or "password" in text_value:
|
|
return "db_auth_failed"
|
|
if "ssl" in name or "tls" in text_value or "certificate" in text_value:
|
|
return "db_tls_failed"
|
|
if str(exc) == "unexpected_result":
|
|
return "unexpected_result"
|
|
return "db_unavailable"
|
|
|
|
|
|
def jsonable(value):
|
|
if isinstance(value, datetime):
|
|
return value.isoformat().replace("+00:00", "Z")
|
|
return value
|
|
|
|
|
|
def create_app(settings: Settings | None = None, probe: Probe | None = None) -> FastAPI:
|
|
cfg = settings or Settings()
|
|
|
|
@asynccontextmanager
|
|
async def lifespan(app: FastAPI):
|
|
app.state.snapshot = Snapshot(
|
|
state="disabled" if not cfg.bitrix_sync_enabled else "starting",
|
|
started_at=utcnow(),
|
|
)
|
|
app.state.stop = asyncio.Event()
|
|
app.state.probe = probe
|
|
app.state.loop_task = None
|
|
if cfg.bitrix_sync_enabled:
|
|
if app.state.probe is None:
|
|
engine = create_async_engine(
|
|
database_url(cfg.bitrix_sync_database_url or ""),
|
|
pool_size=cfg.bitrix_sync_db_pool_size,
|
|
max_overflow=0,
|
|
pool_pre_ping=True,
|
|
pool_recycle=cfg.bitrix_sync_db_pool_recycle_sec,
|
|
connect_args={"server_settings": {"application_name": "han-bitrix-sync"}},
|
|
)
|
|
app.state.probe = DatabaseProbe(engine)
|
|
await run_probe(app)
|
|
app.state.snapshot.worker_running = True
|
|
app.state.loop_task = asyncio.create_task(
|
|
periodic_loop(app), name="db-connectivity-probe"
|
|
)
|
|
yield
|
|
app.state.snapshot.state = "stopping"
|
|
app.state.stop.set()
|
|
if app.state.loop_task:
|
|
app.state.loop_task.cancel()
|
|
with suppress(asyncio.CancelledError):
|
|
await app.state.loop_task
|
|
if app.state.probe:
|
|
await app.state.probe.close()
|
|
|
|
app = FastAPI(
|
|
title="HAN Bitrix Sync Connectivity Stub",
|
|
version="1.0.0",
|
|
lifespan=lifespan,
|
|
docs_url=None if cfg.app_env != "test" else "/docs",
|
|
)
|
|
app.state.settings = cfg
|
|
|
|
@app.middleware("http")
|
|
async def request_id(request: Request, call_next):
|
|
request.state.request_id = request.headers.get("X-Request-ID") or str(uuid.uuid4())
|
|
response = await call_next(request)
|
|
response.headers["X-Request-ID"] = request.state.request_id
|
|
return response
|
|
|
|
@app.exception_handler(HTTPException)
|
|
async def http_error(_: Request, exc: HTTPException):
|
|
return JSONResponse(status_code=exc.status_code, content=exc.detail)
|
|
|
|
@app.get("/health/live")
|
|
async def live():
|
|
return {"status": "live"}
|
|
|
|
@app.get("/health/ready")
|
|
async def ready(request: Request):
|
|
snapshot: Snapshot = request.app.state.snapshot
|
|
if not cfg.bitrix_sync_enabled:
|
|
return JSONResponse({"status": "not_ready", "reason": "sync_disabled"}, status_code=503)
|
|
age = (
|
|
(utcnow() - snapshot.last_success_at).total_seconds()
|
|
if snapshot.last_success_at
|
|
else None
|
|
)
|
|
max_age = max(
|
|
cfg.bitrix_sync_ready_max_staleness_sec,
|
|
2 * cfg.bitrix_sync_db_check_interval_sec * (1 + cfg.bitrix_sync_db_check_jitter_ratio),
|
|
)
|
|
is_ready = (
|
|
snapshot.state == "healthy"
|
|
and snapshot.worker_running
|
|
and age is not None
|
|
and age <= max_age
|
|
)
|
|
if is_ready:
|
|
return {
|
|
"status": "ready",
|
|
"mode": "db_connectivity_stub",
|
|
"database": {
|
|
"status": "ok",
|
|
"last_success_at": jsonable(snapshot.last_success_at),
|
|
"age_seconds": round(age or 0, 3),
|
|
},
|
|
}
|
|
reason = "worker_not_running" if not snapshot.worker_running else "database_unavailable"
|
|
return JSONResponse(
|
|
{
|
|
"status": "not_ready",
|
|
"reason": reason,
|
|
"database": {
|
|
"status": "down",
|
|
"last_success_at": jsonable(snapshot.last_success_at),
|
|
"consecutive_failures": snapshot.consecutive_failures,
|
|
},
|
|
},
|
|
status_code=503,
|
|
)
|
|
|
|
@app.get("/internal/sync/v1/status")
|
|
async def status(
|
|
request: Request,
|
|
authorization: Annotated[str | None, Header()] = None,
|
|
):
|
|
candidate = (
|
|
authorization[7:] if authorization and authorization.startswith("Bearer ") else ""
|
|
)
|
|
if not hmac.compare_digest(candidate, cfg.bitrix_sync_service_token):
|
|
raise HTTPException(
|
|
401,
|
|
{
|
|
"error": {
|
|
"code": "service_unauthorized",
|
|
"message": "Service authentication failed",
|
|
"request_id": request.state.request_id,
|
|
"details": {},
|
|
}
|
|
},
|
|
)
|
|
snapshot: Snapshot = request.app.state.snapshot
|
|
last_check = None
|
|
if snapshot.last_started_at:
|
|
last_check = {
|
|
"started_at": jsonable(snapshot.last_started_at),
|
|
"finished_at": jsonable(snapshot.last_finished_at),
|
|
"success": snapshot.success,
|
|
"duration_ms": snapshot.duration_ms,
|
|
"error_code": snapshot.error_code,
|
|
}
|
|
next_in = (
|
|
max(0, (snapshot.next_check_at - utcnow()).total_seconds())
|
|
if snapshot.next_check_at
|
|
else None
|
|
)
|
|
return {
|
|
"service": "bitrix-sync",
|
|
"enabled": cfg.bitrix_sync_enabled,
|
|
"mode": "db_connectivity_stub",
|
|
"crm_sync_implemented": False,
|
|
"state": snapshot.state,
|
|
"started_at": jsonable(snapshot.started_at),
|
|
"last_check": last_check,
|
|
"last_success_at": jsonable(snapshot.last_success_at),
|
|
"consecutive_failures": snapshot.consecutive_failures,
|
|
"next_check_in_seconds": round(next_in, 3) if next_in is not None else None,
|
|
}
|
|
|
|
return app
|
|
|
|
|
|
async def run_probe(app: FastAPI) -> None:
|
|
cfg: Settings = app.state.settings
|
|
snapshot: Snapshot = app.state.snapshot
|
|
snapshot.last_started_at = utcnow()
|
|
started = time.monotonic()
|
|
try:
|
|
async with asyncio.timeout(cfg.bitrix_sync_db_check_timeout_sec):
|
|
await app.state.probe.check()
|
|
except Exception as exc:
|
|
snapshot.success = False
|
|
snapshot.error_code = classify_error(exc)
|
|
snapshot.consecutive_failures += 1
|
|
snapshot.state = "degraded"
|
|
else:
|
|
snapshot.success = True
|
|
snapshot.error_code = None
|
|
snapshot.consecutive_failures = 0
|
|
snapshot.last_success_at = utcnow()
|
|
snapshot.state = "healthy"
|
|
finally:
|
|
snapshot.last_finished_at = utcnow()
|
|
snapshot.duration_ms = round((time.monotonic() - started) * 1000)
|
|
|
|
|
|
async def periodic_loop(app: FastAPI) -> None:
|
|
cfg: Settings = app.state.settings
|
|
snapshot: Snapshot = app.state.snapshot
|
|
try:
|
|
while not app.state.stop.is_set():
|
|
if snapshot.consecutive_failures:
|
|
cap = min(
|
|
cfg.bitrix_sync_db_retry_base_sec * 2 ** (snapshot.consecutive_failures - 1),
|
|
cfg.bitrix_sync_db_retry_max_sec,
|
|
)
|
|
delay = random.uniform(0, cap)
|
|
else:
|
|
jitter = (
|
|
cfg.bitrix_sync_db_check_interval_sec * cfg.bitrix_sync_db_check_jitter_ratio
|
|
)
|
|
delay = cfg.bitrix_sync_db_check_interval_sec + random.uniform(-jitter, jitter)
|
|
snapshot.next_check_at = utcnow() + timedelta(seconds=delay)
|
|
try:
|
|
await asyncio.wait_for(app.state.stop.wait(), timeout=delay)
|
|
break
|
|
except TimeoutError:
|
|
await run_probe(app)
|
|
finally:
|
|
snapshot.worker_running = False
|
|
|
|
|
|
_settings = Settings()
|
|
app = create_app(_settings)
|
|
|
|
|
|
def run() -> None:
|
|
uvicorn.run("app.main:app", host="0.0.0.0", port=8080)
|