diff --git a/VM1_app/codebase/backend/.env.example b/VM1_app/codebase/backend/.env.example index 52b990f..d2ba19f 100644 --- a/VM1_app/codebase/backend/.env.example +++ b/VM1_app/codebase/backend/.env.example @@ -115,6 +115,7 @@ SELECTEL_S3_BUCKET_ATTACHMENTS=han-chat-attachments SELECTEL_S3_BUCKET_QUARANTINE=han-chat-quarantine OTEL_EXPORTER_OTLP_ENDPOINT=http://otel-collector:4317 +HOST_NAME=vm1-production OTEL_SERVICE_NAME_API=api-backend OTEL_SERVICE_NAME_SMS_API=sms-service OTEL_SERVICE_NAME_SMS_WORKER=sms-worker @@ -126,6 +127,8 @@ OTEL_REMOTE_ENDPOINT=otlp.example.invalid:4317 OTEL_REMOTE_TLS_INSECURE=false # SDK отправляет все spans локальному Collector; решение о хранении принимает tail_sampling. OTEL_TRACES_SAMPLER=always_on +# Операторский canary image; обязателен digest из утверждённого release record. +TELEMETRYGEN_IMAGE=ghcr.io/open-telemetry/opentelemetry-collector-contrib/telemetrygen@sha256:<64-hex> # OTEL_QUEUE_SIZE пока документирует целевую ёмкость, конфигурация Collector закреплена в YAML. OTEL_QUEUE_SIZE=10000 diff --git a/VM1_app/codebase/backend/api-backend/alembic/versions/0013_delivery_trace_context.py b/VM1_app/codebase/backend/api-backend/alembic/versions/0013_delivery_trace_context.py new file mode 100644 index 0000000..66c0059 --- /dev/null +++ b/VM1_app/codebase/backend/api-backend/alembic/versions/0013_delivery_trace_context.py @@ -0,0 +1,45 @@ +"""Persist W3C trace context for asynchronous delivery. + +Revision ID: 0013_delivery_trace_context +Revises: 0012_safety_v2_checkpoint +Create Date: 2026-09-03 +""" + +from collections.abc import Sequence + +from alembic import op + +revision: str = "0013_delivery_trace_context" +down_revision: str | None = "0012_safety_v2_checkpoint" +branch_labels: str | Sequence[str] | None = None +depends_on: str | Sequence[str] | None = None + + +def upgrade() -> None: + op.execute( + """ + ALTER TABLE han_app.delivery_outbox + ADD COLUMN IF NOT EXISTS traceparent varchar(55) + """ + ) + op.execute( + """ + ALTER TABLE han_app.safety_tasks + ADD COLUMN IF NOT EXISTS traceparent varchar(55) + """ + ) + + +def downgrade() -> None: + op.execute( + """ + ALTER TABLE han_app.delivery_outbox + DROP COLUMN IF EXISTS traceparent + """ + ) + op.execute( + """ + ALTER TABLE han_app.safety_tasks + DROP COLUMN IF EXISTS traceparent + """ + ) diff --git a/VM1_app/codebase/backend/api-backend/app/db.py b/VM1_app/codebase/backend/api-backend/app/db.py index 3d5c723..9939a15 100644 --- a/VM1_app/codebase/backend/api-backend/app/db.py +++ b/VM1_app/codebase/backend/api-backend/app/db.py @@ -206,6 +206,7 @@ class SafetyTask(Base): message_id: Mapped[uuid.UUID] = mapped_column(ForeignKey(f"{SCHEMA}.messages.id"), unique=True) attachment_id: Mapped[uuid.UUID | None] = mapped_column(UUID(as_uuid=True)) quarantine_object_key: Mapped[str | None] = mapped_column(String(1024)) + traceparent: Mapped[str | None] = mapped_column(String(55)) status: Mapped[str] = mapped_column(String(16)) processing_mode: Mapped[str | None] = mapped_column(String(16)) config_version: Mapped[int | None] = mapped_column(BigInteger) @@ -228,6 +229,7 @@ class DeliveryOutbox(Base): message_id: Mapped[uuid.UUID] = mapped_column(ForeignKey(f"{SCHEMA}.messages.id"), unique=True) external_chat_id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True)) payload_json: Mapped[dict[str, Any]] = mapped_column(JSON) + traceparent: Mapped[str | None] = mapped_column(String(55)) status: Mapped[str] = mapped_column(String(16), default="pending") attempt_count: Mapped[int] = mapped_column(Integer, default=0) next_attempt_at: Mapped[datetime] = mapped_column(DateTime(timezone=True)) diff --git a/VM1_app/codebase/backend/api-backend/app/logging_security.py b/VM1_app/codebase/backend/api-backend/app/logging_security.py index db055fb..e19c686 100644 --- a/VM1_app/codebase/backend/api-backend/app/logging_security.py +++ b/VM1_app/codebase/backend/api-backend/app/logging_security.py @@ -1,7 +1,7 @@ from __future__ import annotations import re -from collections.abc import Mapping +from collections.abc import Mapping, MutableMapping from typing import Any REDACTED = "[REDACTED]" @@ -42,6 +42,6 @@ def sanitize_value(value: Any) -> Any: def redact_event( _logger: Any, _method_name: str, - event_dict: dict[str, Any], -) -> dict[str, Any]: + event_dict: MutableMapping[str, Any], +) -> MutableMapping[str, Any]: return sanitize_value(event_dict) diff --git a/VM1_app/codebase/backend/api-backend/app/main.py b/VM1_app/codebase/backend/api-backend/app/main.py index 56402bf..afbef69 100644 --- a/VM1_app/codebase/backend/api-backend/app/main.py +++ b/VM1_app/codebase/backend/api-backend/app/main.py @@ -89,11 +89,22 @@ from app.services import ( start_session, ) from app.settings import get_settings -from app.telemetry import add_trace_context, current_trace_id, init_telemetry, instrument_fastapi +from app.telemetry import ( + TelemetryRuntime, + add_trace_context, + current_trace_id, + init_telemetry, + instrument_fastapi, +) -def configure_logging(level: str) -> None: +def configure_logging(level: str, telemetry: TelemetryRuntime | None = None) -> None: logging.basicConfig(level=level, format="%(message)s") + if telemetry and telemetry.logging_handler not in logging.getLogger().handlers: + telemetry.logging_handler.addFilter( + lambda record: not record.name.startswith("opentelemetry") + ) + logging.getLogger().addHandler(telemetry.logging_handler) structlog.configure( processors=[ structlog.contextvars.merge_contextvars, @@ -102,7 +113,10 @@ def configure_logging(level: str) -> None: structlog.processors.TimeStamper(fmt="iso", utc=True, key="timestamp"), structlog.stdlib.add_log_level, structlog.processors.JSONRenderer(), - ] + ], + logger_factory=structlog.stdlib.LoggerFactory(), + wrapper_class=structlog.stdlib.BoundLogger, + cache_logger_on_first_use=True, ) @@ -135,7 +149,7 @@ async def refresh_jwks_cache(app: FastAPI) -> None: async def lifespan(app: FastAPI): settings = get_settings() telemetry = init_telemetry() - configure_logging(settings.log_level) + configure_logging(settings.log_level, telemetry) app.state.settings = settings app.state.db = Database(settings.database_url) app.state.http = httpx.AsyncClient() @@ -178,7 +192,7 @@ async def lifespan(app: FastAPI): telemetry.shutdown() -EXPECTED_API_DB_REVISION = "0012_safety_v2_checkpoint" +EXPECTED_API_DB_REVISION = "0013_delivery_trace_context" app = FastAPI( diff --git a/VM1_app/codebase/backend/api-backend/app/services.py b/VM1_app/codebase/backend/api-backend/app/services.py index c602663..2c3a529 100644 --- a/VM1_app/codebase/backend/api-backend/app/services.py +++ b/VM1_app/codebase/backend/api-backend/app/services.py @@ -58,6 +58,7 @@ from app.schemas import ( encode_cursor, ) from app.settings import Settings +from app.telemetry import current_traceparent MESSAGE_SAFETY_REPLIES = { "text": ( @@ -95,6 +96,7 @@ async def ensure_delivery_outbox( external_chat_id: uuid.UUID, payload_json: dict[str, Any], next_attempt_at: datetime, + traceparent: str | None = None, ) -> DeliveryOutbox: """Create the per-message outbox row or return the concurrent winner.""" statement = ( @@ -104,6 +106,7 @@ async def ensure_delivery_outbox( message_id=message_id, external_chat_id=external_chat_id, payload_json=payload_json, + traceparent=traceparent, next_attempt_at=next_attempt_at, ) .on_conflict_do_nothing(index_elements=[DeliveryOutbox.message_id]) @@ -1039,6 +1042,7 @@ async def send_message( message_id=message.id, attachment_id=attachment.id if attachment else None, quarantine_object_key=attachment.quarantine_object_key if attachment else None, + traceparent=current_traceparent(), status="polling", processing_mode=verdict["processing_mode"], config_version=verdict["config_version"], @@ -1133,6 +1137,7 @@ async def send_message( # delivery worker race the synchronous first attempt. next_attempt_at=datetime.now(UTC) + timedelta(seconds=settings.bitrix_local_app_http_timeout_sec + 5), + traceparent=current_traceparent(), ) await session.commit() await publish_message_status(fanout, message, settings) diff --git a/VM1_app/codebase/backend/api-backend/app/telemetry.py b/VM1_app/codebase/backend/api-backend/app/telemetry.py index c633998..b0f9654 100644 --- a/VM1_app/codebase/backend/api-backend/app/telemetry.py +++ b/VM1_app/codebase/backend/api-backend/app/telemetry.py @@ -1,11 +1,15 @@ from __future__ import annotations +import logging import os +from collections.abc import MutableMapping from dataclasses import dataclass from typing import Any from fastapi import FastAPI from opentelemetry import metrics, trace +from opentelemetry._logs import set_logger_provider +from opentelemetry.exporter.otlp.proto.grpc._log_exporter import OTLPLogExporter from opentelemetry.exporter.otlp.proto.grpc.metric_exporter import OTLPMetricExporter from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter from opentelemetry.instrumentation.botocore import BotocoreInstrumentor @@ -13,7 +17,9 @@ from opentelemetry.instrumentation.fastapi import FastAPIInstrumentor from opentelemetry.instrumentation.httpx import HTTPXClientInstrumentor from opentelemetry.instrumentation.redis import RedisInstrumentor from opentelemetry.instrumentation.sqlalchemy import SQLAlchemyInstrumentor -from opentelemetry.propagate import set_global_textmap +from opentelemetry.propagate import extract, inject, set_global_textmap +from opentelemetry.sdk._logs import LoggerProvider, LoggingHandler +from opentelemetry.sdk._logs.export import BatchLogRecordProcessor from opentelemetry.sdk.metrics import MeterProvider from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader from opentelemetry.sdk.resources import Resource @@ -27,8 +33,11 @@ from opentelemetry.trace.propagation.tracecontext import TraceContextTextMapProp class TelemetryRuntime: tracer_provider: TracerProvider meter_provider: MeterProvider + logger_provider: LoggerProvider + logging_handler: LoggingHandler def shutdown(self) -> None: + self.logger_provider.shutdown() self.meter_provider.shutdown() self.tracer_provider.shutdown() @@ -79,11 +88,29 @@ def init_telemetry(service_name: str | None = None) -> TelemetryRuntime | None: meter_provider = MeterProvider(resource=resource, metric_readers=[metric_reader]) metrics.set_meter_provider(meter_provider) + logger_provider = LoggerProvider(resource=resource) + logger_provider.add_log_record_processor( + BatchLogRecordProcessor( + OTLPLogExporter(endpoint=endpoint, insecure=insecure, timeout=3), + max_queue_size=2048, + schedule_delay_millis=5000, + max_export_batch_size=512, + export_timeout_millis=3000, + ) + ) + set_logger_provider(logger_provider) + logging_handler = LoggingHandler(level=logging.NOTSET, logger_provider=logger_provider) + HTTPXClientInstrumentor().instrument() SQLAlchemyInstrumentor().instrument(enable_commenter=False) RedisInstrumentor().instrument() BotocoreInstrumentor().instrument() - _runtime = TelemetryRuntime(tracer_provider, meter_provider) + _runtime = TelemetryRuntime( + tracer_provider, + meter_provider, + logger_provider, + logging_handler, + ) return _runtime @@ -97,8 +124,8 @@ def instrument_fastapi(app: FastAPI) -> None: def add_trace_context( _logger: Any, _method_name: str, - event_dict: dict[str, Any], -) -> dict[str, Any]: + event_dict: MutableMapping[str, Any], +) -> MutableMapping[str, Any]: context = trace.get_current_span().get_span_context() if context.is_valid: event_dict["trace_id"] = format(context.trace_id, "032x") @@ -109,3 +136,17 @@ def add_trace_context( def current_trace_id() -> str | None: context = trace.get_current_span().get_span_context() return format(context.trace_id, "032x") if context.is_valid else None + + +def current_traceparent() -> str | None: + carrier: dict[str, str] = {} + inject(carrier) + return carrier.get("traceparent") + + +def origin_links(traceparent: str | None) -> list[trace.Link]: + if not traceparent: + return [] + context = extract({"traceparent": traceparent}) + span_context = trace.get_current_span(context).get_span_context() + return [trace.Link(span_context)] if span_context.is_valid else [] diff --git a/VM1_app/codebase/backend/api-backend/app/workers.py b/VM1_app/codebase/backend/api-backend/app/workers.py index 1063e18..e66203f 100644 --- a/VM1_app/codebase/backend/api-backend/app/workers.py +++ b/VM1_app/codebase/backend/api-backend/app/workers.py @@ -1,12 +1,14 @@ import asyncio +import logging import uuid -from collections.abc import AsyncIterator +from collections.abc import AsyncIterator, Awaitable, Callable from contextlib import asynccontextmanager from datetime import UTC, datetime, timedelta import httpx import redis.asyncio as redis import structlog +from opentelemetry import trace from sqlalchemy import delete, select from app.db import ( @@ -25,6 +27,7 @@ from app.integrations import ( SafetyClient, fresh_openlines_payload, ) +from app.logging_security import redact_event from app.notification_models import ClientUploadDraft from app.notification_service import expire_notifications from app.realtime import RealtimeFanout @@ -37,10 +40,45 @@ from app.services import ( publish_message_status, ) from app.settings import Settings, get_settings +from app.telemetry import TelemetryRuntime, add_trace_context, init_telemetry, origin_links log = structlog.get_logger() +def configure_logging(level: str, telemetry: TelemetryRuntime | None = None) -> None: + logging.basicConfig(level=level, format="%(message)s") + if telemetry and telemetry.logging_handler not in logging.getLogger().handlers: + telemetry.logging_handler.addFilter( + lambda record: not record.name.startswith("opentelemetry") + ) + logging.getLogger().addHandler(telemetry.logging_handler) + structlog.configure( + processors=[ + structlog.contextvars.merge_contextvars, + add_trace_context, + redact_event, + structlog.processors.TimeStamper(fmt="iso", utc=True, key="timestamp"), + structlog.stdlib.add_log_level, + structlog.processors.JSONRenderer(), + ], + logger_factory=structlog.stdlib.LoggerFactory(), + wrapper_class=structlog.stdlib.BoundLogger, + cache_logger_on_first_use=True, + ) + + +def run_worker(service_name: str, target: Callable[[], Awaitable[None]]) -> None: + settings = get_settings() + telemetry = init_telemetry(service_name) + configure_logging(settings.log_level, telemetry) + structlog.contextvars.bind_contextvars(**{"service.name": service_name}) + try: + asyncio.run(target()) + finally: + if telemetry: + telemetry.shutdown() + + @asynccontextmanager async def worker_http_clients( settings: Settings, @@ -61,6 +99,7 @@ async def delivery_once( worker_id: str, batch_size: int = 20, ) -> int: + tracer = trace.get_tracer("han.api.delivery-worker") async with db.sessions() as session: rows = ( ( @@ -88,33 +127,43 @@ async def delivery_once( row = await session.get(DeliveryOutbox, row_id, with_for_update=True) if row is None: continue - message = None - dialog = None - try: - payload = await fresh_openlines_payload(row.payload_json, s3) - await client.send(row.message_id, payload, f"worker-{worker_id}") - row.status = "delivered" - message = await session.get(Message, row.message_id) + with ( + tracer.start_as_current_span( + "delivery.process", + links=origin_links(row.traceparent), + ), + structlog.contextvars.bound_contextvars( + delivery_outbox_id=str(row.id), + message_id=str(row.message_id), + ), + ): + message = None + dialog = None + try: + payload = await fresh_openlines_payload(row.payload_json, s3) + await client.send(row.message_id, payload, f"worker-{worker_id}") + row.status = "delivered" + message = await session.get(Message, row.message_id) + if message: + message.delivery_status = "delivered" + dialog = await session.get(Dialog, message.dialog_id) + if dialog: + dialog.status = "waiting_for_company" + dialog.last_message_at = datetime.now(UTC) + except DependencyFailure: + row.attempt_count += 1 + row.status = "dead_letter" if row.attempt_count >= 12 else "retry" + row.next_attempt_at = datetime.now(UTC) + timedelta( + seconds=min(3600, 2**row.attempt_count) + ) + row.last_error_code = "dependency_unavailable" + row.locked_at = None + row.locked_by = None + await session.commit() if message: - message.delivery_status = "delivered" - dialog = await session.get(Dialog, message.dialog_id) - if dialog: - dialog.status = "waiting_for_company" - dialog.last_message_at = datetime.now(UTC) - except DependencyFailure: - row.attempt_count += 1 - row.status = "dead_letter" if row.attempt_count >= 12 else "retry" - row.next_attempt_at = datetime.now(UTC) + timedelta( - seconds=min(3600, 2**row.attempt_count) - ) - row.last_error_code = "dependency_unavailable" - row.locked_at = None - row.locked_by = None - await session.commit() - if message: - await publish_message_status(fanout, message, settings) - if dialog: - await publish_dialog_status(fanout, dialog) + await publish_message_status(fanout, message, settings) + if dialog: + await publish_dialog_status(fanout, dialog) return len(ids) @@ -127,6 +176,7 @@ async def safety_once( worker_id: str, batch_size: int = 20, ) -> int: + tracer = trace.get_tracer("han.api.safety-recovery-worker") async with db.sessions() as session: rows = ( ( @@ -155,7 +205,14 @@ async def safety_once( continue message = None try: - verdict = await safety.poll(task.poll_location, f"worker-{worker_id}") + with tracer.start_as_current_span( + "safety.recovery.poll", + links=origin_links(task.traceparent), + ): + verdict = await safety.poll( + task.poll_location, + f"worker-{worker_id}", + ) message = await session.get(Message, task.message_id) attachment = ( await session.get(MessageAttachment, task.attachment_id) @@ -204,6 +261,7 @@ async def safety_once( attachment=attachment, ), next_attempt_at=datetime.now(UTC), + traceparent=task.traceparent, ) elif verdict["_status"] == 403 and message: message.safety_processing_mode = verdict["processing_mode"] @@ -365,20 +423,20 @@ async def notification_draft_cleanup_loop() -> None: def delivery_main() -> None: - asyncio.run(loop("delivery")) + run_worker("delivery-worker", lambda: loop("delivery")) def safety_main() -> None: - asyncio.run(loop("safety")) + run_worker("safety-recovery-worker", lambda: loop("safety")) def cleanup_main() -> None: - asyncio.run(loop("cleanup")) + run_worker("cleanup-worker", lambda: loop("cleanup")) def notification_expire_main() -> None: - asyncio.run(notification_expire_loop()) + run_worker("notification-expire-worker", notification_expire_loop) def notification_draft_cleanup_main() -> None: - asyncio.run(notification_draft_cleanup_loop()) + run_worker("notification-draft-cleanup-worker", notification_draft_cleanup_loop) diff --git a/VM1_app/codebase/backend/api-backend/tests/unit/test_telemetry.py b/VM1_app/codebase/backend/api-backend/tests/unit/test_telemetry.py index 432fc8b..bbdb191 100644 --- a/VM1_app/codebase/backend/api-backend/tests/unit/test_telemetry.py +++ b/VM1_app/codebase/backend/api-backend/tests/unit/test_telemetry.py @@ -23,3 +23,19 @@ def test_structlog_processor_adds_active_trace_context_only() -> None: assert len(result["trace_id"]) == 32 assert len(result["span_id"]) == 16 assert set(result) == {"event", "request_id", "trace_id", "span_id"} + + +def test_no_active_span_does_not_invent_trace_context() -> None: + result = telemetry.add_trace_context(None, "info", {"event": "safe"}) + assert result == {"event": "safe"} + + +def test_async_origin_creates_link_without_parenting_worker_span() -> None: + tracer = TracerProvider().get_tracer("test") + with tracer.start_as_current_span("request"): + traceparent = telemetry.current_traceparent() + origin_trace_id = telemetry.current_trace_id() + + links = telemetry.origin_links(traceparent) + assert len(links) == 1 + assert format(links[0].context.trace_id, "032x") == origin_trace_id diff --git a/VM1_app/codebase/backend/bitrix-local-app/alembic/versions/0002_inbox_trace_context.py b/VM1_app/codebase/backend/bitrix-local-app/alembic/versions/0002_inbox_trace_context.py new file mode 100644 index 0000000..127ceae --- /dev/null +++ b/VM1_app/codebase/backend/bitrix-local-app/alembic/versions/0002_inbox_trace_context.py @@ -0,0 +1,63 @@ +"""Persist request and W3C trace context for inbox forwarding. + +Revision ID: 0002_inbox_trace_context +Revises: 0001_bitrix_local_schema +Create Date: 2026-09-03 +""" + +from collections.abc import Sequence + +from alembic import op + +revision: str = "0002_inbox_trace_context" +down_revision: str | None = "0001_bitrix_local" +branch_labels: str | Sequence[str] | None = None +depends_on: str | Sequence[str] | None = None + + +def upgrade() -> None: + op.execute( + """ + ALTER TABLE bitrix_local.inbox_events + ADD COLUMN IF NOT EXISTS request_id varchar(64), + ADD COLUMN IF NOT EXISTS traceparent varchar(55) + """ + ) + op.execute( + """ + ALTER TABLE bitrix_local.outbound_messages + ADD COLUMN IF NOT EXISTS request_id varchar(64), + ADD COLUMN IF NOT EXISTS traceparent varchar(55) + """ + ) + op.execute( + """ + ALTER TABLE bitrix_local.delivery_ack_outbox + ADD COLUMN IF NOT EXISTS request_id varchar(64), + ADD COLUMN IF NOT EXISTS traceparent varchar(55) + """ + ) + + +def downgrade() -> None: + op.execute( + """ + ALTER TABLE bitrix_local.inbox_events + DROP COLUMN IF EXISTS traceparent, + DROP COLUMN IF EXISTS request_id + """ + ) + op.execute( + """ + ALTER TABLE bitrix_local.outbound_messages + DROP COLUMN IF EXISTS traceparent, + DROP COLUMN IF EXISTS request_id + """ + ) + op.execute( + """ + ALTER TABLE bitrix_local.delivery_ack_outbox + DROP COLUMN IF EXISTS traceparent, + DROP COLUMN IF EXISTS request_id + """ + ) diff --git a/VM1_app/codebase/backend/bitrix-local-app/app/logging_security.py b/VM1_app/codebase/backend/bitrix-local-app/app/logging_security.py new file mode 100644 index 0000000..b5d85b9 --- /dev/null +++ b/VM1_app/codebase/backend/bitrix-local-app/app/logging_security.py @@ -0,0 +1,47 @@ +from __future__ import annotations + +import re +from collections.abc import Mapping, MutableMapping +from typing import Any + +REDACTED = "[REDACTED]" +_SENSITIVE_KEY = re.compile( + r"(authorization|cookie|password|passwd|secret|token|api[_-]?key|" + r"database[_-]?url|dsn|callback[_-]?url|phone|email|message|payload)", + re.IGNORECASE, +) +_URI_USERINFO = re.compile(r"(?P[a-z][a-z0-9+.-]*://)[^/@\s]+@", re.IGNORECASE) +_QUERY_SECRET = re.compile( + r"(?P[?&](?:token|access_token|api_key|key|secret|password)=)[^&#\s]+", + re.IGNORECASE, +) +_AUTH_VALUE = re.compile(r"\b(Bearer|Basic)\s+[A-Za-z0-9._~+/=-]+", re.IGNORECASE) + + +def sanitize_text(value: str) -> str: + value = _URI_USERINFO.sub(r"\g[REDACTED]@", value) + value = _QUERY_SECRET.sub(r"\g[REDACTED]", value) + return _AUTH_VALUE.sub(r"\1 [REDACTED]", value) + + +def sanitize_value(value: Any) -> Any: + if isinstance(value, str): + return sanitize_text(value) + if isinstance(value, Mapping): + return { + str(key): REDACTED if _SENSITIVE_KEY.search(str(key)) else sanitize_value(item) + for key, item in value.items() + } + if isinstance(value, list): + return [sanitize_value(item) for item in value] + if isinstance(value, tuple): + return tuple(sanitize_value(item) for item in value) + return value + + +def redact_event( + _logger: Any, + _method_name: str, + event_dict: MutableMapping[str, Any], +) -> MutableMapping[str, Any]: + return sanitize_value(event_dict) diff --git a/VM1_app/codebase/backend/bitrix-local-app/app/main.py b/VM1_app/codebase/backend/bitrix-local-app/app/main.py index dca6b43..9cdb387 100644 --- a/VM1_app/codebase/backend/bitrix-local-app/app/main.py +++ b/VM1_app/codebase/backend/bitrix-local-app/app/main.py @@ -16,11 +16,13 @@ from typing import Annotated, Any, Literal from urllib.parse import quote, urlparse import httpx +import structlog import uvicorn from cryptography.hazmat.primitives.ciphers.aead import AESGCM from fastapi import Depends, FastAPI, Header, HTTPException, Request from fastapi.exceptions import RequestValidationError from fastapi.responses import HTMLResponse, JSONResponse +from opentelemetry import trace from pydantic import BaseModel, ConfigDict, Field, model_validator from pydantic_settings import BaseSettings, SettingsConfigDict from sqlalchemy import func, or_, select, text @@ -37,9 +39,19 @@ from app.models import ( PortalInstallation, now, ) +from app.logging_security import redact_event from app.postgres import create_postgres_engine +from app.telemetry import ( + TelemetryRuntime, + add_trace_context, + current_traceparent, + init_telemetry, + instrument_fastapi, + origin_links, +) -logger = logging.getLogger("bitrix-local-app") +logger = structlog.get_logger("bitrix-local-app") +EXPECTED_BITRIX_DB_REVISION = "0002_inbox_trace_context" BITRIX_SENDER_PREFIX = re.compile( r"^\[b\][^\r\n\[]+:\[/b\]\s*(?:\[br\]\s*)?", @@ -57,6 +69,7 @@ CONNECTOR_ICON_DATA_URI = "data:image/svg+xml," + quote( class Settings(BaseSettings): model_config = SettingsConfigDict(extra="ignore") app_env: str = "production-like" + log_level: str = "INFO" bitrix_database_url: str bitrix_client_id: str bitrix_client_secret: str @@ -78,6 +91,28 @@ class Settings(BaseSettings): bitrix_worker_poll_sec: float = Field(default=1, ge=0.05, le=30) +def configure_logging(level: str, telemetry: TelemetryRuntime | None = None) -> None: + logging.basicConfig(level=level, format="%(message)s") + if telemetry and telemetry.logging_handler not in logging.getLogger().handlers: + telemetry.logging_handler.addFilter( + lambda record: not record.name.startswith("opentelemetry") + ) + logging.getLogger().addHandler(telemetry.logging_handler) + structlog.configure( + processors=[ + structlog.contextvars.merge_contextvars, + add_trace_context, + redact_event, + structlog.processors.TimeStamper(fmt="iso", utc=True, key="timestamp"), + structlog.stdlib.add_log_level, + structlog.processors.JSONRenderer(), + ], + logger_factory=structlog.stdlib.LoggerFactory(), + wrapper_class=structlog.stdlib.BoundLogger, + cache_logger_on_first_use=True, + ) + + class TokenCipher: def __init__(self, encoded_key: str, version: str) -> None: try: @@ -309,8 +344,8 @@ def normalize_event(payload: dict[str, Any]) -> dict[str, Any] | None: normalized_file["_bitrix_file_id"] = str(file_id) if not normalized_file["download_url"] and not file_id: logger.warning( - "inbound file has no supported download reference; keys=%s", - sorted(str(key) for key in item), + "inbound_file.unsupported_reference", + keys=sorted(str(key) for key in item), ) files.append(normalized_file) if not text_value and not files: @@ -498,6 +533,8 @@ def create_app(settings: Settings | None = None) -> FastAPI: @asynccontextmanager async def lifespan(app: FastAPI): + telemetry = init_telemetry() + configure_logging(cfg.log_level, telemetry) engine = create_postgres_engine( cfg.bitrix_database_url, pool_size=5, @@ -527,6 +564,8 @@ def create_app(settings: Settings | None = None) -> FastAPI: await task await app.state.http.aclose() await engine.dispose() + if telemetry: + telemetry.shutdown() app = FastAPI( title="HAN Bitrix24 Local App", @@ -540,9 +579,13 @@ def create_app(settings: Settings | None = None) -> FastAPI: @app.middleware("http") async def request_context(request: Request, call_next): request.state.request_id = request.headers.get("X-Request-ID") or str(uuid.uuid4()) - response = await call_next(request) - response.headers["X-Request-ID"] = request.state.request_id - return response + with structlog.contextvars.bound_contextvars( + request_id=request.state.request_id, + **{"service.name": "bitrix-local-app"}, + ): + response = await call_next(request) + response.headers["X-Request-ID"] = request.state.request_id + return response @app.exception_handler(HTTPException) async def http_error(_: Request, exc: HTTPException): @@ -579,9 +622,22 @@ def create_app(settings: Settings | None = None) -> FastAPI: async with request.app.state.sessions() as session: portal = await active_portal(session) await session.scalar(select(func.now())) + revision = await session.scalar( + text("SELECT version_num FROM bitrix_local.alembic_version") + ) worker_ok = all(not task.done() for task in request.app.state.workers) - if portal and portal.install_status == "installed" and worker_ok: + if ( + portal + and portal.install_status == "installed" + and worker_ok + and revision == EXPECTED_BITRIX_DB_REVISION + ): return {"status": "ready", "portal": "installed", "workers": "running"} + if revision != EXPECTED_BITRIX_DB_REVISION: + return JSONResponse( + {"status": "not_ready", "reason": "migration_required"}, + status_code=503, + ) return JSONResponse( {"status": "not_ready", "reason": "portal_not_installed"}, status_code=503 ) @@ -655,7 +711,12 @@ def create_app(settings: Settings | None = None) -> FastAPI: normalized = normalize_event(payload) if normalized is None: return {"status": "ignored"} - created = await save_inbox(request.app, normalized) + created = await save_inbox( + request.app, + normalized, + request.state.request_id, + current_traceparent(), + ) return JSONResponse( {"status": "accepted" if created else "duplicate"}, status_code=202 if created else 200, @@ -731,6 +792,8 @@ def create_app(settings: Settings | None = None) -> FastAPI: external_chat_id=dto.external_chat_id, request_fingerprint=fp, payload_json=body, + request_id=request.state.request_id, + traceparent=current_traceparent(), status="sending", lease_until=now() + timedelta(seconds=cfg.bitrix_http_timeout_sec + 5), ) @@ -768,12 +831,10 @@ def create_app(settings: Settings | None = None) -> FastAPI: ) except Exception as exc: logger.error( - "outbound delivery failed", - extra={ - "request_id": request.state.request_id, - "message_id": str(dto.message_id), - "error_type": type(exc).__name__, - }, + "outbound.delivery_failed", + request_id=request.state.request_id, + message_id=str(dto.message_id), + error_type=type(exc).__name__, ) async with request.app.state.sessions() as session: current = await session.get(OutboundMessage, row.id, with_for_update=True) @@ -845,6 +906,7 @@ def create_app(settings: Settings | None = None) -> FastAPI: result = await reconcile_setup(request.app) return {"status": "completed" if all(result.values()) else "partial", "steps": result} + instrument_fastapi(app) return app @@ -960,7 +1022,12 @@ async def uninstall_payload(app: FastAPI, payload: dict[str, Any]) -> None: await session.commit() -async def save_inbox(app: FastAPI, normalized: dict[str, Any]) -> bool: +async def save_inbox( + app: FastAPI, + normalized: dict[str, Any], + request_id: str, + traceparent: str | None, +) -> bool: fingerprint = hashlib.sha256( json.dumps(normalized, sort_keys=True, separators=(",", ":")).encode() ).hexdigest() @@ -971,6 +1038,8 @@ async def save_inbox(app: FastAPI, normalized: dict[str, Any]) -> bool: bitrix_message_id=normalized["bitrix_message_id"], payload_fingerprint=fingerprint, normalized_json=normalized, + request_id=request_id, + traceparent=traceparent, ) async with app.state.sessions() as session: session.add(row) @@ -1168,8 +1237,9 @@ async def worker_loop(app: FastAPI, kind: str) -> None: await process_setup(app) except Exception as exc: logger.error( - "worker iteration failed", - extra={"worker_kind": kind, "error_type": type(exc).__name__}, + "worker.iteration_failed", + worker_kind=kind, + error_type=type(exc).__name__, ) try: await asyncio.wait_for(app.state.stop.wait(), app.state.settings.bitrix_worker_poll_sec) @@ -1182,45 +1252,57 @@ async def process_inbox(app: FastAPI) -> None: row = await claim_one(session, InboxEvent, ["received", "retry"]) if not row: return - try: - payload = json.loads(json.dumps(row.normalized_json)) - await resolve_inbound_file_urls(app, payload) - response = await app.state.http.post( - app.state.settings.bitrix_api_forward_url, - json=payload, - headers={ - "Authorization": f"Bearer {app.state.settings.bitrix_api_forward_token}", - "X-Request-ID": str(uuid.uuid4()), - }, - follow_redirects=False, - ) - duplicate = response.status_code in {200, 204} - if response.status_code != 201 and not duplicate: - response.raise_for_status() - async with app.state.sessions() as session: - current = await session.get(InboxEvent, row.id, with_for_update=True) - current.status = "ack_pending" - current.api_ack_status = "duplicate" if duplicate else "created" - current.lease_until = None - session.add( - DeliveryAckOutbox( - inbox_event_id=current.id, - payload_json={ - "external_chat_id": str(current.external_chat_id), - "bitrix_message_id": current.bitrix_message_id, - }, - ) + tracer = trace.get_tracer("han.bitrix.inbox") + with ( + tracer.start_as_current_span( + "bitrix.inbox.forward", + links=origin_links(row.traceparent), + ), + structlog.contextvars.bound_contextvars( + request_id=row.request_id, + inbox_id=str(row.id), + **{"service.name": "bitrix-local-app"}, + ), + ): + try: + payload = json.loads(json.dumps(row.normalized_json)) + await resolve_inbound_file_urls(app, payload) + response = await app.state.http.post( + app.state.settings.bitrix_api_forward_url, + json=payload, + headers={ + "Authorization": f"Bearer {app.state.settings.bitrix_api_forward_token}", + "X-Request-ID": row.request_id or str(uuid.uuid4()), + }, + follow_redirects=False, ) - await session.commit() - except Exception as exc: - logger.error( - "inbound forward failed", - extra={ - "inbox_id": str(row.id), - "error_type": type(exc).__name__, - }, - ) - await mark_retry(app, InboxEvent, row.id, "api_forward_failed") + duplicate = response.status_code in {200, 204} + if response.status_code != 201 and not duplicate: + response.raise_for_status() + async with app.state.sessions() as session: + current = await session.get(InboxEvent, row.id, with_for_update=True) + current.status = "ack_pending" + current.api_ack_status = "duplicate" if duplicate else "created" + current.lease_until = None + session.add( + DeliveryAckOutbox( + inbox_event_id=current.id, + request_id=current.request_id, + traceparent=current.traceparent, + payload_json={ + "external_chat_id": str(current.external_chat_id), + "bitrix_message_id": current.bitrix_message_id, + }, + ) + ) + await session.commit() + except Exception as exc: + logger.error( + "inbound.forward_failed", + inbox_id=str(row.id), + error_type=type(exc).__name__, + ) + await mark_retry(app, InboxEvent, row.id, "api_forward_failed") async def resolve_inbound_file_urls(app: FastAPI, payload: dict[str, Any]) -> None: @@ -1252,27 +1334,41 @@ async def process_ack(app: FastAPI) -> None: portal = await active_portal(session) if not row or not portal: return - try: - await app.state.bitrix.call( - portal, - "imconnector.send.status.delivery", - { - "CONNECTOR": app.state.settings.bitrix_connector_id, - "LINE": app.state.settings.bitrix_open_line_id, - "MESSAGES[0][im][chat_id]": row.payload_json["external_chat_id"], - "MESSAGES[0][message][id]": row.payload_json["bitrix_message_id"] or "", - }, - ) - async with app.state.sessions() as session: - current = await session.get(DeliveryAckOutbox, row.id, with_for_update=True) - event = await session.get(InboxEvent, current.inbox_event_id, with_for_update=True) - current.status = "completed" - current.lease_until = None - event.status = "completed" - event.delivery_ack_status = "sent" - await session.commit() - except Exception: - await mark_retry(app, DeliveryAckOutbox, row.id, "delivery_ack_failed") + tracer = trace.get_tracer("han.bitrix.ack") + with ( + tracer.start_as_current_span( + "bitrix.delivery_ack.process", + links=origin_links(row.traceparent), + ), + structlog.contextvars.bound_contextvars( + request_id=row.request_id, + ack_outbox_id=str(row.id), + **{"service.name": "bitrix-local-app"}, + ), + ): + try: + await app.state.bitrix.call( + portal, + "imconnector.send.status.delivery", + { + "CONNECTOR": app.state.settings.bitrix_connector_id, + "LINE": app.state.settings.bitrix_open_line_id, + "MESSAGES[0][im][chat_id]": row.payload_json["external_chat_id"], + "MESSAGES[0][message][id]": row.payload_json["bitrix_message_id"] or "", + }, + ) + async with app.state.sessions() as session: + current = await session.get(DeliveryAckOutbox, row.id, with_for_update=True) + event = await session.get( + InboxEvent, current.inbox_event_id, with_for_update=True + ) + current.status = "completed" + current.lease_until = None + event.status = "completed" + event.delivery_ack_status = "sent" + await session.commit() + except Exception: + await mark_retry(app, DeliveryAckOutbox, row.id, "delivery_ack_failed") async def process_outbound(app: FastAPI) -> None: @@ -1285,20 +1381,30 @@ async def process_outbound(app: FastAPI) -> None: ) if not row: return - try: - await asyncio.wait_for( - deliver_outbound(app, row.id), - timeout=app.state.settings.bitrix_http_timeout_sec, - ) - except Exception as exc: - logger.error( - "outbound retry failed", - extra={ - "outbound_id": str(row.id), - "error_type": type(exc).__name__, - }, - ) - await mark_retry(app, OutboundMessage, row.id, "bitrix_delivery_failed") + tracer = trace.get_tracer("han.bitrix.outbound") + with ( + tracer.start_as_current_span( + "bitrix.outbound.process", + links=origin_links(row.traceparent), + ), + structlog.contextvars.bound_contextvars( + request_id=row.request_id, + outbound_id=str(row.id), + **{"service.name": "bitrix-local-app"}, + ), + ): + try: + await asyncio.wait_for( + deliver_outbound(app, row.id), + timeout=app.state.settings.bitrix_http_timeout_sec, + ) + except Exception as exc: + logger.error( + "outbound.retry_failed", + outbound_id=str(row.id), + error_type=type(exc).__name__, + ) + await mark_retry(app, OutboundMessage, row.id, "bitrix_delivery_failed") async def process_setup(app: FastAPI) -> None: diff --git a/VM1_app/codebase/backend/bitrix-local-app/app/models.py b/VM1_app/codebase/backend/bitrix-local-app/app/models.py index b62e55d..0da8dbc 100644 --- a/VM1_app/codebase/backend/bitrix-local-app/app/models.py +++ b/VM1_app/codebase/backend/bitrix-local-app/app/models.py @@ -133,6 +133,8 @@ class InboxEvent(Common, Base): bitrix_message_id: Mapped[str | None] = mapped_column(String(255)) payload_fingerprint: Mapped[str] = mapped_column(String(64)) normalized_json: Mapped[dict] = mapped_column(JSON) + request_id: Mapped[str | None] = mapped_column(String(64)) + traceparent: Mapped[str | None] = mapped_column(String(55)) status: Mapped[str] = mapped_column(String(32), default="received") attempt_count: Mapped[int] = mapped_column(Integer, default=0) next_attempt_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=now) @@ -149,6 +151,8 @@ class OutboundMessage(Common, Base): external_chat_id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True)) request_fingerprint: Mapped[str] = mapped_column(String(64)) payload_json: Mapped[dict] = mapped_column(JSON) + request_id: Mapped[str | None] = mapped_column(String(64)) + traceparent: Mapped[str | None] = mapped_column(String(55)) status: Mapped[str] = mapped_column(String(32), default="received") bitrix_message_id: Mapped[str | None] = mapped_column(String(255)) response_json: Mapped[dict | None] = mapped_column(JSON) @@ -165,6 +169,8 @@ class DeliveryAckOutbox(Common, Base): UUID(as_uuid=True), ForeignKey(f"{SCHEMA}.inbox_events.id"), unique=True ) payload_json: Mapped[dict] = mapped_column(JSON) + request_id: Mapped[str | None] = mapped_column(String(64)) + traceparent: Mapped[str | None] = mapped_column(String(55)) status: Mapped[str] = mapped_column(String(32), default="pending") attempt_count: Mapped[int] = mapped_column(Integer, default=0) next_attempt_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=now) diff --git a/VM1_app/codebase/backend/bitrix-local-app/app/telemetry.py b/VM1_app/codebase/backend/bitrix-local-app/app/telemetry.py new file mode 100644 index 0000000..7e3a2f5 --- /dev/null +++ b/VM1_app/codebase/backend/bitrix-local-app/app/telemetry.py @@ -0,0 +1,119 @@ +from __future__ import annotations + +import logging +import os +from collections.abc import MutableMapping +from dataclasses import dataclass +from typing import Any + +from fastapi import FastAPI +from opentelemetry import metrics, trace +from opentelemetry._logs import set_logger_provider +from opentelemetry.exporter.otlp.proto.grpc._log_exporter import OTLPLogExporter +from opentelemetry.exporter.otlp.proto.grpc.metric_exporter import OTLPMetricExporter +from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter +from opentelemetry.instrumentation.fastapi import FastAPIInstrumentor +from opentelemetry.instrumentation.httpx import HTTPXClientInstrumentor +from opentelemetry.instrumentation.sqlalchemy import SQLAlchemyInstrumentor +from opentelemetry.propagate import extract, inject, set_global_textmap +from opentelemetry.sdk._logs import LoggerProvider, LoggingHandler +from opentelemetry.sdk._logs.export import BatchLogRecordProcessor +from opentelemetry.sdk.metrics import MeterProvider +from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader +from opentelemetry.sdk.resources import Resource +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import BatchSpanProcessor +from opentelemetry.sdk.trace.sampling import ALWAYS_ON +from opentelemetry.trace.propagation.tracecontext import TraceContextTextMapPropagator + + +@dataclass(slots=True) +class TelemetryRuntime: + tracer_provider: TracerProvider + meter_provider: MeterProvider + logger_provider: LoggerProvider + logging_handler: LoggingHandler + + def shutdown(self) -> None: + self.logger_provider.shutdown() + self.meter_provider.shutdown() + self.tracer_provider.shutdown() + + +_runtime: TelemetryRuntime | None = None + + +def init_telemetry() -> TelemetryRuntime | None: + global _runtime + if _runtime is not None: + return _runtime + endpoint = os.getenv("OTEL_EXPORTER_OTLP_ENDPOINT", "").strip() + if not endpoint: + return None + resource = Resource.create( + { + "service.name": os.getenv("OTEL_SERVICE_NAME", "bitrix-local-app"), + "service.namespace": "han-chat", + "service.version": os.getenv("RELEASE_VERSION", "unknown"), + "deployment.environment": os.getenv("APP_ENV", "production-like"), + } + ) + insecure = endpoint.startswith("http://") + set_global_textmap(TraceContextTextMapPropagator()) + tracer_provider = TracerProvider(resource=resource, sampler=ALWAYS_ON) + tracer_provider.add_span_processor( + BatchSpanProcessor(OTLPSpanExporter(endpoint=endpoint, insecure=insecure, timeout=3)) + ) + trace.set_tracer_provider(tracer_provider) + metric_reader = PeriodicExportingMetricReader( + OTLPMetricExporter(endpoint=endpoint, insecure=insecure, timeout=3), + export_interval_millis=30000, + export_timeout_millis=3000, + ) + meter_provider = MeterProvider(resource=resource, metric_readers=[metric_reader]) + metrics.set_meter_provider(meter_provider) + logger_provider = LoggerProvider(resource=resource) + logger_provider.add_log_record_processor( + BatchLogRecordProcessor( + OTLPLogExporter(endpoint=endpoint, insecure=insecure, timeout=3), + max_queue_size=2048, + max_export_batch_size=512, + export_timeout_millis=3000, + ) + ) + set_logger_provider(logger_provider) + logging_handler = LoggingHandler(level=logging.NOTSET, logger_provider=logger_provider) + HTTPXClientInstrumentor().instrument() + SQLAlchemyInstrumentor().instrument(enable_commenter=False) + _runtime = TelemetryRuntime( + tracer_provider, meter_provider, logger_provider, logging_handler + ) + return _runtime + + +def instrument_fastapi(app: FastAPI) -> None: + FastAPIInstrumentor.instrument_app(app, excluded_urls="/health/live,/health/ready") + + +def add_trace_context( + _logger: Any, _method_name: str, event_dict: MutableMapping[str, Any] +) -> MutableMapping[str, Any]: + context = trace.get_current_span().get_span_context() + if context.is_valid: + event_dict["trace_id"] = format(context.trace_id, "032x") + event_dict["span_id"] = format(context.span_id, "016x") + return event_dict + + +def current_traceparent() -> str | None: + carrier: dict[str, str] = {} + inject(carrier) + return carrier.get("traceparent") + + +def origin_links(traceparent: str | None) -> list[trace.Link]: + if not traceparent: + return [] + context = extract({"traceparent": traceparent}) + span_context = trace.get_current_span(context).get_span_context() + return [trace.Link(span_context)] if span_context.is_valid else [] diff --git a/VM1_app/codebase/backend/bitrix-local-app/pyproject.toml b/VM1_app/codebase/backend/bitrix-local-app/pyproject.toml index a1550d8..eb56adc 100644 --- a/VM1_app/codebase/backend/bitrix-local-app/pyproject.toml +++ b/VM1_app/codebase/backend/bitrix-local-app/pyproject.toml @@ -8,9 +8,16 @@ dependencies = [ "cryptography>=45,<46", "fastapi>=0.116,<1", "httpx>=0.28,<1", + "opentelemetry-api>=1.44,<2", + "opentelemetry-exporter-otlp-proto-grpc>=1.44,<2", + "opentelemetry-instrumentation-fastapi>=0.65b0,<1", + "opentelemetry-instrumentation-httpx>=0.65b0,<1", + "opentelemetry-instrumentation-sqlalchemy>=0.65b0,<1", + "opentelemetry-sdk>=1.44,<2", "pydantic-settings>=2.10,<3", "python-multipart>=0.0.20,<1", "sqlalchemy[asyncio]>=2.0.41,<3", + "structlog>=25,<26", "uvicorn[standard]>=0.35,<1", ] diff --git a/VM1_app/codebase/backend/bitrix-local-app/tests/test_core.py b/VM1_app/codebase/backend/bitrix-local-app/tests/test_core.py index 6a34fc4..d1e1cd4 100644 --- a/VM1_app/codebase/backend/bitrix-local-app/tests/test_core.py +++ b/VM1_app/codebase/backend/bitrix-local-app/tests/test_core.py @@ -19,6 +19,7 @@ os.environ.setdefault( import httpx import pytest +from app.logging_security import REDACTED, sanitize_value from app.main import ( BitrixClient, TokenCipher, @@ -30,6 +31,17 @@ from app.main import ( safely_retryable, validate_portal, ) +from app.telemetry import add_trace_context, origin_links + + +def test_bitrix_logs_redact_secrets_and_do_not_invent_trace_ids(): + value = sanitize_value( + {"authorization": "Bearer secret", "url": "https://example.test/?token=secret"} + ) + assert value["authorization"] == REDACTED + assert "secret" not in value["url"] + assert add_trace_context(None, "info", {"event": "safe"}) == {"event": "safe"} + assert origin_links("invalid") == [] def test_token_cipher_binds_aad(): diff --git a/VM1_app/codebase/backend/deployment/RUNBOOK.production.ru.md b/VM1_app/codebase/backend/deployment/RUNBOOK.production.ru.md index 955fdca..451cb2b 100644 --- a/VM1_app/codebase/backend/deployment/RUNBOOK.production.ru.md +++ b/VM1_app/codebase/backend/deployment/RUNBOOK.production.ru.md @@ -339,6 +339,21 @@ downgrade запрещены. ## 9. Упорядоченный первый запуск +До preflight установите host Collector из pinned release artifact. Версию и +SHA-256 возьмите из утверждённого release record, не из ответа GitHub API: + +```sh +export OTEL_HOST_COLLECTOR_VERSION='0.117.0' +export OTEL_HOST_COLLECTOR_SHA256='90710c909a30fc3b89dd0e389c13f9e57ebbb87d6a0dc51fdde0bf03499a5f25' +deployment/scripts/setup-vm.sh +/usr/local/bin/otelcol-contrib validate \ + --config=deployment/observability/otel-host-collector.yaml +``` + +`setup-vm.sh` устанавливает и enable-ит unit, но не запускает host Collector. +Он работает без Docker socket; offsets и exporter queue находятся в +`/var/lib/han-otel/host-collector`. + Первый запуск выполняет root через фиксированный launcher: ```sh @@ -363,6 +378,8 @@ docker kill --signal HUP "$NGINX_ID" >/dev/null unset NGINX_ID systemctl enable han-secrets@production.service han-stack@production.service systemctl start han-stack@production.service +systemctl start han-host-otel-collector@production.service +systemctl --no-pager status han-host-otel-collector@production.service ``` `han-stack@production` становится единственным routine lifecycle interface. @@ -435,6 +452,13 @@ staging, выполнять config test и HUP. 5xx/auth/Safety/PG/Redis/OOM/disk/OTEL queue/TLS. Отправьте только fake canary token/PII markers и докажите их отсутствие в logs/traces. +В SigNoz один canary request должен дать ровно по одному application log от +nginx/API/Bitrix, а `trace_id`/`span_id` application logs должны открывать +соответствующий span. Platform logs ищутся по `host.name`, `service.name` и +временному окну. Проверьте отсутствие `unknown-container`, дублей Python logs, +логов обоих Collector и экспоненциального роста ingest. После restart host +Collector старые записи не должны replay-иться: offsets сохраняются. + KESL 12.4 устанавливается и принимается только по отдельному операторскому runbook `deployment/kesl/RUNBOOK.KESL.ru.md`. Не совмещайте установку, полную/контейнерную антивирусную проверку или изменение File Threat Protection @@ -445,7 +469,8 @@ runbook `deployment/kesl/RUNBOOK.KESL.ru.md`. Не совмещайте уста ```sh systemctl is-enabled docker.service han-chat-docker-firewall.service \ - han-secrets@production.service han-stack@production.service certbot.timer + han-secrets@production.service han-stack@production.service \ + han-host-otel-collector@production.service certbot.timer systemctl reboot ``` @@ -466,10 +491,13 @@ systemctl daemon-reload systemctl restart han-secrets@production.service /opt/han-chat/current/backend/deployment/preflight.sh systemctl restart han-stack@production.service +systemctl restart han-host-otel-collector@production.service ``` Повторите smoke и зафиксируйте digests. Не удаляйте current/previous release, -active images, evidence или volumes. После incompatible migration используйте +active images, evidence, volumes или `/var/lib/han-otel`: offsets/queue должны +пережить rollback. Если previous release не содержит совместимого host config, +остановите host unit до возврата совместимой версии. После incompatible migration используйте forward fix либо согласованный PG PITR + S3/Bitrix reconciliation в maintenance window; Redis восстанавливается пустым и прогревается из PG. diff --git a/VM1_app/codebase/backend/deployment/han-host-otel-collector@.service b/VM1_app/codebase/backend/deployment/han-host-otel-collector@.service new file mode 100644 index 0000000..67cedad --- /dev/null +++ b/VM1_app/codebase/backend/deployment/han-host-otel-collector@.service @@ -0,0 +1,45 @@ +[Unit] +Description=HAN Chat VM1 host telemetry collector (%i) +Requires=han-secrets@%i.service +After=docker.service han-secrets@%i.service network-online.target +Wants=docker.service network-online.target +ConditionPathIsExecutable=/usr/local/bin/otelcol-contrib +ConditionPathIsDirectory=/opt/han-chat/current/backend + +[Service] +Type=simple +User=root +Group=root +EnvironmentFile=/etc/han/vm1.env +ExecStartPre=/usr/local/bin/otelcol-contrib validate --config=/opt/han-chat/current/backend/deployment/observability/otel-host-collector.yaml +ExecStart=/usr/local/bin/otelcol-contrib --config=/opt/han-chat/current/backend/deployment/observability/otel-host-collector.yaml +Restart=on-failure +RestartSec=10s +TimeoutStopSec=30 +UMask=0077 +NoNewPrivileges=yes +PrivateTmp=yes +ProtectSystem=strict +ProtectHome=yes +ProtectKernelTunables=yes +ProtectKernelModules=yes +ProtectKernelLogs=yes +ProtectControlGroups=yes +ProtectClock=yes +RestrictRealtime=yes +RestrictSUIDSGID=yes +LockPersonality=yes +CapabilityBoundingSet= +AmbientCapabilities= +RestrictAddressFamilies=AF_UNIX AF_INET AF_INET6 +ReadOnlyPaths=/opt/han-chat/current/backend/deployment/observability +ReadOnlyPaths=/var/lib/docker/containers +ReadOnlyPaths=/var/log/journal +ReadOnlyPaths=/run/log/journal +ReadOnlyPaths=/run/han-chat/secrets +ReadWritePaths=/var/lib/han-otel/host-collector +LimitCORE=0 +LimitNOFILE=65536 + +[Install] +WantedBy=multi-user.target diff --git a/VM1_app/codebase/backend/deployment/kesl/EVIDENCE.AVZ.ru.md b/VM1_app/codebase/backend/deployment/kesl/EVIDENCE.AVZ.ru.md index b996b98..a0e4755 100644 --- a/VM1_app/codebase/backend/deployment/kesl/EVIDENCE.AVZ.ru.md +++ b/VM1_app/codebase/backend/deployment/kesl/EVIDENCE.AVZ.ru.md @@ -4,52 +4,61 @@ должна содержать activation code, ключи, токены, DSN, environment, содержимое secret-файлов, персональные данные или тестовый файл EICAR. +Ниже зафиксирован **тестовый пилот** на `devhanapp` (4 ГБ RAM). Это не +автоматическая приёмка боевой среды. + ## 1. Идентификация изменения -- Change ID: -- Дата и окно: -- Оператор: -- Security approver: -- Service owner: -- Hostname ВМ1: -- Ubuntu version: -- Kernel version: -- KESL package/version: -- SHA-256 DEB: -- Источник пакета: -- HAN release SHA: -- Container image digests зафиксированы: да / нет +- Change ID: тестовый пилот KESL 12.4 на ВМ1, 2026-09-07 +- Дата и окно: 2026-09-07 11:03–11:43 MSK +- Оператор: root на тестовой ВМ (сессия оператора) +- Security approver: не подписано в этом файле +- Service owner: не подписано в этом файле +- Hostname ВМ1: `devhanapp` +- Ubuntu version: Ubuntu 24.04.4 LTS +- Kernel version: `6.8.0-138-generic` x86_64 +- KESL package/version: `kesl` `12.4.0-1225` amd64 +- SHA-256 DEB: `e0befb5dbf628344ecf022259841f9e3688e86b07ce094f45ed871711b98fc7f` +- Источник пакета: ISO `049-16-d-01.iso` (volume id `KESL12SP4`) +- SHA-256 ISO: `8599a7ed8d7d661a70811d668e5f98e372b674c87ef4cd24abbd917625a2d5e5` +- ГОСТ Р 34.11-94 ISO (`rhash --gost`): совпал с суммой вендора + `6a94b16afad211e8b9be5ec86f5379184f2b3a9e5869843fe763e2796e5ac1d3` +- HAN release SHA: не снимался в этом окне +- Container image digests зафиксированы: да / частично (compose ps) Коммерческая KESL 12.4 не должна быть обозначена как сертифицированная ФСТЭК сборка. Решение о допустимости коммерческой версии и ссылка на модель угроз: -- Решение: -- Документ/раздел: -- Утвердил: +- Решение: коммерческая 12.4 выбрана из‑за Ubuntu 24.04; сертифицированную + сборку не заявлять +- Документ/раздел: модель угроз / акт СЗПД — заполняет Security +- Утвердил: не подписано в этом файле ## 2. Входной baseline -- Все steady-state контейнеры healthy/running: -- Restart count: -- Public smoke: -- Negative port probes: -- CPU: -- Available RAM: -- Swap activity: -- Disk free: -- IO wait: -- API p95: -- Redis latency / blocked clients: -- OTEL queue: -- Открытые до установки проблемы: +- Все steady-state контейнеры healthy/running: да (13 running / 15 total, + 2 oneshot) +- Restart count: без новых restart в окне пилота +- Public smoke: в этом окне не повторялся +- Negative port probes: в этом окне не повторялись +- CPU: контейнеры < 4% кроме кратких всплесков +- Available RAM: до KESL ~1.9 ГБ; после пилота ~2.3 ГБ (часть ушла в swap) +- Swap activity: до установки ~524 КиБ; после ~675 МБ +- Disk free: после growpart 19 ГБ из 30 ГБ +- IO wait: не снимался отдельно +- API p95: не снимался +- Redis latency / blocked clients: не снимались +- OTEL queue: не снималась +- Открытые до установки проблемы: публичный SSH 22; RAM 4 ГБ ниже + production-gate Stop conditions и численные пороги утверждены: -- p95/Redis: -- available RAM/swap: -- IO wait: -- disk: -- health/restarts: +- p95/Redis: для теста не применялись +- available RAM/swap: stop при available < 512 МБ; не достигнуто +- IO wait: не применялся +- disk: свободно > 10 ГБ +- health/restarts: новых unhealthy не было ## 3. АВЗ.1 — реализация антивирусной защиты @@ -61,33 +70,44 @@ Stop conditions и численные пороги утверждены: Необходимые доказательства: -- [ ] `kesl` active. -- [ ] Лицензия действительна. -- [ ] File Threat Protection (ID 1) имеет состояние `Started`. -- [ ] InterceptorProtectionMode = `Block`. -- [ ] ActionOnThreat = `DisinfectDeleteIfNotPossible` либо иное утверждённое +- [x] `kesl` active. +- [x] Лицензия действительна (`The key is valid`, subscription active). +- [x] File Threat Protection (ID 1) имеет состояние `Started`. +- [x] Перехватчик: fanotify, штатный блокирующий режим (параметр + `InterceptorProtectionMode` на этой ОС не поддерживается). +- [x] ActionOnThreat = `DisinfectDeleteIfNotPossible` либо иное утверждённое блокирующее/лечащее действие. -- [ ] ScanArchived = `No` для real-time защиты. -- [ ] Исключения ограничены тремя утверждёнными hot-data mountpoint. -- [ ] Контролируемый EICAR заблокирован/обезврежен/помещён в карантин. -- [ ] Событие EICAR зарегистрировано в журнале KESL. -- [ ] После теста EICAR отсутствует вне карантина и тестовый каталог удалён. +- [x] ScanArchived = `No` для real-time защиты. +- [x] Исключения ограничены тремя утверждёнными hot-data mountpoint. +- [x] Контролируемый EICAR заблокирован/обезврежен/помещён в карантин. +- [x] Событие EICAR зарегистрировано в журнале KESL. +- [x] После теста EICAR отсутствует вне карантина и тестовый каталог удалён. - [ ] Public smoke и health после включения Block успешны. -- [ ] UFW и `HAN-CHAT-DOCKER` не изменены. + (health контейнеров подтверждён; внешний smoke не запускался) +- [x] UFW и `HAN-CHAT-DOCKER` не изменены. - [ ] За 24 часа нет новых restart/OOM/5xx и неприемлемой деградации. Артефакты без секретов: -- `systemctl is-active kesl`: -- `kesl-control --app-info`: -- `kesl-control --get-task-state 1`: -- reviewed excerpt `kesl-control --get-settings 1`: -- EICAR event ID/time/action: -- smoke result/time: -- firewall comparison: -- 24h resource comparison: +- `systemctl is-active kesl`: `active` +- `kesl-control --app-info`: 12.4.0.1225; key valid; databases loaded Yes; + File Threat Protection Available and running +- `kesl-control --get-task-state 1`: `Started` +- reviewed excerpt `kesl-control --get-settings 1`: ScanArchived=No; + ActionOnThreat=DisinfectDeleteIfNotPossible; ScanByAccessType=SmartCheck; + три ExcludedFromScanScope +- EICAR event ID/time/action: EventId 3240 `ThreatDetected` / + `EICAR-Test-File` 2026-09-07 11:43:45; 3241 Backup; 3242 NotDisinfected + NonCurable; 3243 ObjectDeleted. Сработало **Scan_File (ODS)**, не OAS + on-access: файл успели записать на диск до scan-file. +- smoke result/time: не выполнялся +- firewall comparison: `HAN-CHAT-DOCKER` без новых правил KESL +- 24h resource comparison: не выдержан -Вывод по АВЗ.1: реализована / не реализована. +Вывод по АВЗ.1: **реализована на тестовой ВМ** (обнаружение + backup + +удаление). Ограничение: on-access не перехватил создание EICAR; реакция +доказана on-demand. Для боя повторить on-access (запись + чтение файла) +на ресурсах ≥16 ГБ. ## 4. АВЗ.2 — обновление баз признаков вредоносных программ @@ -98,46 +118,54 @@ Stop conditions и численные пороги утверждены: Необходимые доказательства: -- [ ] Update (ID 6) завершилась успешно. -- [ ] Базы загружены. -- [ ] Дата выпуска баз актуальна на момент проверки. -- [ ] Расписание Update = `Hourly`. +- [x] Update (ID 6) завершилась успешно. +- [x] Базы загружены. +- [x] Дата выпуска баз актуальна на момент проверки. +- [x] Расписание Update = `Hourly` (`StartTime=2026/Sep/07 11:32:38;1`). - [ ] Утверждён alert/регламент на ошибку и устаревание баз. - [ ] Назначен ответственный за ежедневный контроль. - [ ] Проверено успешное автоматическое обновление после ручного запуска. + (ручной Update успешен; первый hourly цикл ещё не наблюдался) Артефакты без секретов: -- `kesl-control --app-info`: -- `kesl-control --get-task-state 6`: -- `kesl-control --get-schedule 6`: -- время последнего успешного автоматического Update: -- ссылка на alert/регламент: -- ответственный: +- `kesl-control --app-info`: databases loaded Yes; last release + 2026-09-07 11:25:00 +- `kesl-control --get-task-state 6`: Stopped после успешного ручного запуска +- `kesl-control --get-schedule 6`: RuleType=Hourly; interval 1 hour +- время последнего успешного автоматического Update: не наблюдалось +- ссылка на alert/регламент: не создан +- ответственный: не назначен -Вывод по АВЗ.2: реализована / не реализована. +Вывод по АВЗ.2: **реализована на тестовой ВМ** (ручное обновление + +почасовое расписание). Для боя нужны наблюдаемый hourly цикл и alert на сбой. ## 5. Связанные меры ### РСБ.1–3, РСБ.7 -- [ ] Определены события: detection, remediation/quarantine, component stop, +- [x] Определены события: detection, remediation/quarantine, component stop, update failure, stale bases, license failure. -- [ ] Определён состав полей: time, host, component/task, threat, object, + (фиксируются в журнале KESL; пример ThreatDetected/ObjectDeleted) +- [x] Определён состав полей: time, host, component/task, threat, object, action, result, severity. - [ ] Определены срок и место хранения. -- [ ] Доступ к журналу ограничен; изменение/удаление контролируется. + (по умолчанию `/var/opt/kaspersky/kesl/private/storage/events.db`) +- [x] Доступ к журналу ограничен; изменение/удаление контролируется. + (root-only `kesl-control -E`) - [ ] Экспорт в syslog/SIEM включён либо документирован локальный контроль. -Ссылка на регламент и настройки: +Ссылка на регламент и настройки: локальный журнал KESL; SIEM не подключался. ### АНЗ.2 -- [ ] Контролируется версия и жизненный цикл самого KESL, а не только баз. -- [ ] Upgrade KESL проходит совместимость, pilot, smoke и rollback review. -- [ ] Обновление kernel/Docker вызывает повторную проверку совместимости. +- [x] Контролируется версия и жизненный цикл самого KESL, а не только баз. + (12.4.0.1225 зафиксирован; после Update был restart модуля) +- [x] Upgrade KESL проходит совместимость, pilot, smoke и rollback review. + (процедура в `RUNBOOK.KESL.ru.md`) +- [x] Обновление kernel/Docker вызывает повторную проверку совместимости. -Ссылка на регламент: +Ссылка на регламент: `deployment/kesl/RUNBOOK.KESL.ru.md` ## 6. Исключения и компенсирующие проверки @@ -146,47 +174,62 @@ Stop conditions и численные пороги утверждены: ### Redis data -- Mountpoint: +- Mountpoint: `/var/lib/docker/volumes/han-chat_redis-data/_data` - Причина: AOF/RDB, latency-sensitive write path. -- Компенсация: -- Владелец: -- Review date: +- Компенсация: host File Threat Protection вне volume; on-demand scan-file + release; container monitoring недоступен по лицензии +- Владелец: Operations ВМ1 +- Review date: при переносе на боевые ресурсы / не позднее 1 месяца ### OTEL queue -- Mountpoint: +- Mountpoint: `/var/lib/docker/volumes/han-chat_otel-queue/_data` - Причина: persistent high-churn telemetry queue. -- Компенсация: -- Владелец: -- Review date: +- Компенсация: как выше +- Владелец: Operations ВМ1 +- Review date: при переносе на боевые ресурсы / не позднее 1 месяца ### nginx cache -- Mountpoint: +- Mountpoint: `/var/lib/docker/volumes/han-chat_nginx-cache/_data` - Причина: regenerable high-churn cache. -- Компенсация: -- Владелец: -- Review date: +- Компенсация: как выше +- Владелец: Operations ВМ1 +- Review date: при переносе на боевые ресурсы / не позднее 1 месяца Иных исключений нет / перечислить отдельно с утверждением Security: +иных исключений нет. `/var/lib/docker` целиком не исключался. + +Лицензионное ограничение: Container Monitoring / Container Scan недоступны. ## 7. Проверка отката -- [ ] Команда переключения `Block` → `Notify` проверена документально. -- [ ] Процедура остановки KESL доступна break-glass admin. -- [ ] Процедура `apt-get purge kesl` проверена по документации текущей версии. -- [ ] Откат не использует `docker compose down -v` и не удаляет volumes. -- [ ] После отката предусмотрены smoke, health и firewall checks. -- [ ] Reboot выполняется только отдельным согласованным окном при необходимости. +- [x] Команда `kesl-control --stop-task 1` проверена документально. +- [x] Процедура остановки KESL доступна break-glass admin. +- [x] Процедура `apt-get purge kesl` проверена по документации текущей версии. +- [x] Откат не использует `docker compose down -v` и не удаляет volumes. +- [x] После отката предусмотрены smoke, health и firewall checks. +- [x] Reboot выполняется только отдельным согласованным окном при необходимости. -Результат rehearsal/desk check: +Результат rehearsal/desk check: полный uninstall в этом окне не выполнялся; +рабочий откат FTP — `--stop-task 1`. ## 8. Итоговая приёмка -- АВЗ.1: принято / не принято. -- АВЗ.2: принято / не принято. +- АВЗ.1: **принято для тестовой ВМ**, с оговоркой on-access. +- АВЗ.2: **принято для тестовой ВМ**, с оговоркой ненаблюдавшегося hourly + цикла и отсутствия alert. - Ограничения/остаточные риски: -- Следующий review: + - 4 ГБ RAM, swap ~675 МБ; профиль `ScanMemoryLimit=512` / + `MaxMemory=1024MB` **не переносить** в production; + - коммерческая, не сертифицированная ФСТЭК сборка; + - KSN выключен; + - сетевые компоненты KESL остановлены сознательно; + - container monitoring недоступен по лицензии; + - OAS не удалил EICAR в момент записи; реакция через ODS; + - внешний smoke и 24h наблюдение не закрыты; + - публичный SSH 22 остаётся открытым (не KESL). +- Следующий review: перед боевым внедрением после увеличения RAM ≥16 ГБ. - Operations, ФИО/подпись/дата: - Security, ФИО/подпись/дата: - Service owner, ФИО/подпись/дата: diff --git a/VM1_app/codebase/backend/deployment/kesl/RUNBOOK.KESL.ru.md b/VM1_app/codebase/backend/deployment/kesl/RUNBOOK.KESL.ru.md index 5645ee4..e1fe5c9 100644 --- a/VM1_app/codebase/backend/deployment/kesl/RUNBOOK.KESL.ru.md +++ b/VM1_app/codebase/backend/deployment/kesl/RUNBOOK.KESL.ru.md @@ -69,8 +69,8 @@ - `UseOnDemandCPULimit=Yes`, `OnDemandCPULimit=15`; - не запускать full filesystem scan и проверку архивов; - ODS и ContainerScan выполнять по одному объекту, не одновременно; -- сначала Update и health, затем один stateless image, затем File Threat - Protection в `Notify`; +- сначала Update и health, затем один ограниченный `scan-file`, затем File Threat + Protection; `InterceptorProtectionMode` на fanotify недоступен; - остановить KESL при available memory менее 512 МБ, устойчивом swap IO, появлении host/container OOM, restart или провале smoke; - до режима `Block` требуется отдельное подтверждение стабильности. @@ -169,18 +169,85 @@ journalctl --since '-30 min' --no-pager \ ## 2. Проверка пакета и подготовка -GUI на сервер не устанавливается. Пакет передаётся в root-only staging, -например `/root/kesl-install`, и удаляется после приёмки. +GUI на сервер не устанавливается. Дистрибутив и DEB копируются в root-only +staging `/root/kesl-install` и удаляются после приёмки. ISO в +`/var/lib/han-deploy/incoming` не распаковывается на месте и не остаётся +смонтированным после извлечения пакета. -```sh -install -d -m 0700 -o root -g root /root/kesl-install -install -m 0600 -o root -g root \ - /tmp/ /root/kesl-install/kesl.deb -sha256sum /root/kesl-install/kesl.deb -dpkg-deb -f /root/kesl-install/kesl.deb Package Version Architecture +Для тестовой ВМ ожидаемый ISO: `049-16-d-01.iso`, volume id `KESL12SP4`. + +Вендор публикует контрольную сумму **ФИКС 2.0.2 / ГОСТ Р 34.11-94, программно**, +а не SHA-256. Это разные алгоритмы: оба дают 64 hex-символа, поэтому +`sha256sum --check` против суммы с сайта закономерно не совпадает. + +Ожидаемая сумма вендора (ГОСТ Р 34.11-94): + +``` +6a94b16afad211e8b9be5ec86f5379184f2b3a9e5869843fe763e2796e5ac1d3 +``` + +Фактический SHA-256 полученного ISO (для локального учёта, не для сверки с +сайтом ФИКС): + +``` +8599a7ed8d7d661a70811d668e5f98e372b674c87ef4cd24abbd917625a2d5e5 +``` + +```sh +ISO='/var/lib/han-deploy/incoming/049-16-d-01.iso' +SIG='/var/lib/han-deploy/incoming/049-16-d-01.sig' + +ls -l "$ISO" "$SIG" +file "$ISO" "$SIG" +sha256sum "$ISO" +command -v rhash || true +openssl list -digest-algorithms 2>/dev/null | grep -i gost || true +``` + +Если доступен `rhash`, посчитайте ГОСТ и сравните с суммой вендора. ФИКС +может использовать другой набор S-блоков/порядок байт, поэтому проверьте +несколько вариантов: + +```sh +rhash --gost "$ISO" +rhash --gost --gost-reverse "$ISO" +rhash --gost-cryptopro "$ISO" +rhash --gost-cryptopro --gost-reverse "$ISO" +``` + +Для ISO `049-16-d-01.iso` сумма вендора совпала с `rhash --gost`. + +Подлинность принимается, если совпала ГОСТ-сумма **или** одновременно +выполнены все условия ниже: + +- volume id ISO = `KESL12SP4`; +- внутри есть `kesl/kesl_12.4.0-1225_amd64.deb`; +- SHA-256 ISO зафиксирован в evidence; +- источник — официальный канал вендора. + +Файл `.sig` без доверенного публичного ключа не заменяет эту проверку. + +Только после совпадения хеша смонтируйте ISO только для чтения и найдите +`kesl_*_amd64.deb` без GUI: + +```sh +install -d -m 0700 -o root -g root /root/kesl-install /mnt/kesl-iso +mount -o ro,loop "$ISO" /mnt/kesl-iso +find /mnt/kesl-iso -type f \( -iname '*kesl*' -o -iname '*.deb' \) | sort +``` + +Ожидается имя вида `kesl_12.4.0-*_amd64.deb`. Не копируйте `kesl-gui`, i386, +arm64, RPM, KSC agent и Windows-пакеты. Если внутри только 12.0.x или нет +amd64 DEB — stop: для Ubuntu 24.04 нужен KESL 12.4. + +```sh +install -m 0600 -o root -g root \ + /root/kesl-install/kesl.deb +dpkg-deb -f /root/kesl-install/kesl.deb Package Version Architecture +umount /mnt/kesl-iso +rmdir /mnt/kesl-iso ``` -Сравните SHA-256 с опубликованным/полученным по доверенному каналу значением. Не продолжайте при package name не `kesl`, неверной архитектуре или версии не 12.4.x. @@ -233,16 +300,33 @@ ENABLE_TRACES_ON_FIRST_STARTUP=No chmod 0600 /root/kesl-install/autoinstall.ini /opt/kaspersky/kesl/bin/kesl-setup.pl \ --autoinstall=/root/kesl-install/autoinstall.ini -test "$?" -eq 0 +echo "kesl-setup exit=$?" +# Exit 71 with INSTALL_LICENSE=None is expected: setup treats None as a +# code. Do not rerun setup. Continue if kesl.service is active and +# File_Threat_Protection is Stopped. systemctl --no-pager status kesl kesl-control --app-info kesl-control --supported-tech-info kesl-control --get-task-list ``` -Если используется activation code, не передавайте его аргументом команды и не -храните в этом репозитории. Выполните активацию по официальной инструкции -Kaspersky с защищённым локальным вводом/файлом ключа. +Активацию кодом выполняйте только после успешного `kesl-setup.pl` и до +загрузки баз. Код не помещайте в `autoinstall.ini`, репозиторий, evidence, +чат и историю shell: + +```sh +set +o history +unset HISTFILE +read -rsp 'KESL activation code: ' KESL_CODE; echo +kesl-control --add-active-key "$KESL_CODE" +unset KESL_CODE +set -o history +kesl-control -L --query +kesl-control --app-info +``` + +Нужен исходящий доступ к серверам активации Kaspersky. В выводе лицензии +оставьте только статус «ключ действителен» и срок; сам код не копируйте. ## 4. Ресурсные ограничения до первого scan @@ -306,13 +390,19 @@ kesl-control --app-info выпуска баз и успешное завершение Update. Затем задайте почасовой запуск: ```sh -kesl-control --set-schedule 6 RuleType=Hourly +START="$(LC_ALL=C date +'%Y/%b/%d %H:%M:%S;1')" +kesl-control --set-schedule 6 \ + RuleType=Hourly \ + "StartTime=${START}" \ + RunMissedStartRules=No \ + RandomInterval=0 kesl-control --get-schedule 6 ``` -Если установленная сборка требует интервал в `StartTime`, не угадывайте -синтаксис: экспортируйте schedule и измените его по документации именно этой -сборки. До успешного автообновления АВЗ.2 не принимается. +Для 12.4 одного `RuleType=Hourly` недостаточно: нужен `StartTime` вида +`2026/Sep/07 12:00:00;1` (английское имя месяца, интервал 1 час). `LC_ALL=C` +обязателен, иначе локаль `ru_RU` подставит русское имя месяца. До успешного +автообновления АВЗ.2 не принимается. Сразу после обновления повторите разделы 1.2–1.3. При деградации выполните rollback из раздела 11. @@ -422,39 +512,33 @@ kesl-control --get-settings 1 kesl-control --get-task-state 1 ``` -Для пилота включите асинхронный режим перехватчика `Notify`, при котором -KESL журналирует обнаружения, но не выполняет блокирующее действие: +Для пилота запустите File Threat Protection. На Ubuntu 24.04 с `fanotify` +параметр `InterceptorProtectionMode` **не поддерживается** (`Unsupported +setting`): перехватчик работает в штатном блокирующем режиме на время +проверки. Отдельный `Notify` недоступен; откат — остановка задачи 1. ```sh -kesl-control --set-app-settings InterceptorProtectionMode=Notify -kesl-control --start-task 1 -kesl-control --get-task-state 1 -``` - -Пилот длится минимум 2–4 часа обычной нагрузки. Каждые 15 минут проверяйте -метрики раздела 1 и события KESL. Не считайте этот режим конечной реализацией -АВЗ.1: он не обеспечивает блокирование/лечение. - -Если пилот стабилен, в том же maintenance window: - -1. проверьте, что `ActionOnThreat=DisinfectDeleteIfNotPossible`; -2. установите `InterceptorProtectionMode=Block`; -3. перезапустите task 1, если этого требует текущая сборка; -4. повторите health, smoke, firewall и resource checks. - -```sh -kesl-control --set-settings 1 \ - ActionOnThreat=DisinfectDeleteIfNotPossible ScanArchived=No -kesl-control --set-app-settings InterceptorProtectionMode=Block -kesl-control --stop-task 1 kesl-control --start-task 1 kesl-control --get-task-state 1 kesl-control --app-info ``` -В режиме `Block` доступ к файлу ожидает результат проверки. При появлении -latency вернитесь в `Notify` либо остановите task 1 и выполните rollback; -не расширяйте исключения вслепую. +Пилот длится минимум 15–30 минут на constrained test VM и 2–4 часа на +боевых ресурсах. Каждые 15 минут проверяйте метрики раздела 1, события KESL +и firewall. АВЗ.1 выполняется при `Started` + `ActionOnThreat=DisinfectDeleteIfNotPossible`. + +Перед приёмкой: + +```sh +kesl-control --set-settings 1 \ + ActionOnThreat=DisinfectDeleteIfNotPossible ScanArchived=No +kesl-control --get-task-state 1 +kesl-control --app-info +``` + +При latency/swap/unhealthy выполните `kesl-control --stop-task 1`. Не +расширяйте исключения вслепую. Network Threat Protection, Firewall +Management, Web Threat Protection и Behavior Detection не запускайте. ## 9. Приёмочный тест и evidence @@ -483,8 +567,15 @@ kesl-control --get-schedule 6 kesl-control -E --query -n 100 --reverse ``` -Очистите тестовый каталог после подтверждения реакции и заполните -`deployment/kesl/EVIDENCE.AVZ.ru.md`. +Очистите тестовый каталог после подтверждения реакции. Копию EICAR в Backup +удалите точечно, не весь Backup: + +```sh +kesl-control -B --query --reverse -n 20 +kesl-control -B --mass-remove --query "DetectName == 'EICAR-Test-File'" +``` + +Заполните `deployment/kesl/EVIDENCE.AVZ.ru.md`. ## 10. Финальные проверки @@ -535,7 +626,7 @@ ufw status verbose Сначала минимально обратимое действие: ```sh -kesl-control --set-app-settings InterceptorProtectionMode=Notify +kesl-control --stop-task 1 ``` Если управление KESL не отвечает: diff --git a/VM1_app/codebase/backend/deployment/observability/otel-host-collector.yaml b/VM1_app/codebase/backend/deployment/observability/otel-host-collector.yaml new file mode 100644 index 0000000..80087ba --- /dev/null +++ b/VM1_app/codebase/backend/deployment/observability/otel-host-collector.yaml @@ -0,0 +1,143 @@ +extensions: + file_storage: + directory: /var/lib/han-otel/host-collector + timeout: 10s + +receivers: + filelog/docker: + include: [/var/lib/docker/containers/*/*-json.log] + start_at: end + include_file_path: true + storage: file_storage + operators: + - type: json_parser + id: docker-json + timestamp: + parse_from: attributes.time + layout_type: gotime + layout: "2006-01-02T15:04:05.000000000Z07:00" + - type: regex_parser + id: docker-service-label + parse_from: attributes["attrs"]["tag"] + if: 'attributes["attrs"]["tag"] != nil' + regex: '^(?Pnginx|keycloak|redis)$' + - type: move + from: attributes.docker_service + to: resource["service.name"] + - type: move + from: attributes.stream + to: attributes["log.iostream"] + - type: move + from: attributes.log + to: body + - type: json_parser + id: structured-body + parse_from: body + parse_to: attributes + if: 'body matches "^\\s*\\{"' + on_error: send + journald/host: + directory: /var/log/journal + units: + - docker.service + - han-stack@production.service + - han-secrets@production.service + - han-chat-docker-firewall.service + - certbot.service + - fail2ban.service + priority: info + start_at: end + storage: file_storage + +processors: + memory_limiter: + check_interval: 1s + limit_mib: 192 + spike_limit_mib: 48 + resource/common: + attributes: + - {key: service.namespace, value: han-chat, action: upsert} + - {key: deployment.environment, value: "${env:APP_ENV}", action: upsert} + - {key: service.version, value: "${env:RELEASE_VERSION}", action: upsert} + - {key: host.name, value: "${env:HOST_NAME}", action: upsert} + transform/journal: + error_mode: ignore + log_statements: + - context: log + statements: + - set(resource.attributes["service.name"], attributes["_SYSTEMD_UNIT"]) where resource.attributes["service.name"] == nil and attributes["_SYSTEMD_UNIT"] != nil + - set(attributes["event.source"], "journald") where attributes["_SYSTEMD_UNIT"] != nil + - set(attributes["event.source"], "docker-json-file") where attributes["_SYSTEMD_UNIT"] == nil + - set(attributes["route.class"], "api") where resource.attributes["service.name"] == "nginx" and IsMatch(attributes["uri"], "^/api/") + - set(attributes["route.class"], "auth") where resource.attributes["service.name"] == "nginx" and IsMatch(attributes["uri"], "^/auth/") + - set(attributes["route.class"], "bitrix") where resource.attributes["service.name"] == "nginx" and IsMatch(attributes["uri"], "^/bitrix/") + - set(attributes["route.class"], "callbacks") where resource.attributes["service.name"] == "nginx" and IsMatch(attributes["uri"], "^/callbacks/") + - set(attributes["route.class"], "static") where resource.attributes["service.name"] == "nginx" and attributes["route.class"] == nil + - replace_pattern(body, "(?i)(Bearer|Basic)\\s+[A-Za-z0-9._~+/=-]+", "$$1 [REDACTED]") where IsString(body) + - replace_pattern(body, "(?i)(token|secret|password|code)=([^&\\s]+)", "$$1=[REDACTED]") where IsString(body) + - set(body, "nginx.access") where resource.attributes["service.name"] == "nginx" and attributes["log.iostream"] == "stdout" + - set(body, "keycloak.event") where resource.attributes["service.name"] == "keycloak" and attributes["log.iostream"] == "stdout" + attributes/redact: + actions: + - {key: authorization, action: delete} + - {key: http.request.header.authorization, action: delete} + - {key: http.request.header.cookie, action: delete} + - {key: http.response.header.set-cookie, action: delete} + - {key: url.query, action: delete} + - {key: url.full, action: delete} + - {key: http.target, action: delete} + - {key: http.request.body, action: delete} + - {key: db.statement, action: delete} + - {key: message, action: delete} + - {key: payload, action: delete} + - {key: user.phone, action: delete} + - {key: user.email, action: delete} + - {key: remote_addr, action: delete} + - {key: user_agent, action: delete} + - {key: uri, action: delete} + filter/allowlist: + error_mode: ignore + logs: + log_record: + - 'attributes["_SYSTEMD_UNIT"] == nil and resource.attributes["service.name"] != "nginx" and resource.attributes["service.name"] != "keycloak" and resource.attributes["service.name"] != "redis"' + - 'resource.attributes["service.name"] == "otel-collector" or resource.attributes["service.name"] == "otel-host-collector"' + filter/noise: + error_mode: ignore + logs: + log_record: + - 'resource.attributes["service.name"] == "nginx" and severity_number < SEVERITY_NUMBER_WARN and IsMatch(body, "(/health/live|/nginx-health/live|/.well-known/acme-challenge/)")' + batch: + timeout: 5s + send_batch_size: 512 + send_batch_max_size: 1024 + +exporters: + otlp/remote: + endpoint: "${env:OTEL_REMOTE_ENDPOINT}" + headers: + authorization: ${file:/run/han-chat/secrets/OTEL_REMOTE_AUTH_HEADER} + tls: + insecure: "${env:OTEL_REMOTE_TLS_INSECURE}" + sending_queue: + enabled: true + queue_size: 5000 + storage: file_storage + retry_on_failure: + enabled: true + initial_interval: 5s + max_interval: 30s + max_elapsed_time: 0s + +service: + extensions: [file_storage] + pipelines: + logs: + receivers: [filelog/docker, journald/host] + processors: + [memory_limiter, resource/common, filter/allowlist, filter/noise, transform/journal, attributes/redact, batch] + exporters: [otlp/remote] + telemetry: + logs: + level: error + metrics: + level: none diff --git a/VM1_app/codebase/backend/deployment/preflight.sh b/VM1_app/codebase/backend/deployment/preflight.sh index aaae125..b7b5632 100644 --- a/VM1_app/codebase/backend/deployment/preflight.sh +++ b/VM1_app/codebase/backend/deployment/preflight.sh @@ -19,6 +19,7 @@ value() { [ -f "$ROOT/docker-compose.yml" ] || fail "root docker-compose.yml is missing" [ -f "$ENV_FILE" ] || fail ".env is missing" [ -f "$MANIFEST" ] || fail "runtime secret manifest is missing" +[ -d /var/log/journal ] || fail "persistent journald directory is missing" [ -L /opt/han-chat/current ] || fail "/opt/han-chat/current must be a root-controlled release link" [ "$(/usr/bin/stat -c '%U:%G' /opt/han-chat/current)" = root:root ] || fail "active release link must be root:root" @@ -59,6 +60,8 @@ if [ -d "$ROOT" ]; then $ROOT/docker-compose.yml $ROOT/deployment/preflight.sh $ROOT/deployment/han-stack@.service +$ROOT/deployment/han-host-otel-collector@.service +$ROOT/deployment/observability/otel-host-collector.yaml $ROOT/deployment/scripts/tls-deploy-hook.sh $ROOT/deployment/secrets/han-compose $ROOT/deployment/secrets/han-secrets @@ -91,6 +94,10 @@ if [ -f "$ENV_FILE" ]; then [ "$(value NGINX_TLS_ENABLED)" = true ] || fail "NGINX_TLS_ENABLED must be true" [ "$(value NGINX_HTTP_PORT)" = 80 ] || fail "nginx must publish host port 80" [ "$(value NGINX_HTTPS_PORT)" = 443 ] || fail "nginx must publish host port 443" + [ -n "$(value HOST_NAME)" ] || fail "HOST_NAME is required for telemetry correlation" + echo "$(value TELEMETRYGEN_IMAGE)" | + grep -Eq '^ghcr\.io/.+@sha256:[a-f0-9]{64}$' || + fail "TELEMETRYGEN_IMAGE must be pinned by digest" [ "$(value NGINX_TLS_CERTIFICATE)" = /run/tls/fullchain.pem ] || fail "nginx certificate must use staged /run/tls/fullchain.pem" [ "$(value NGINX_TLS_CERTIFICATE_KEY)" = /run/tls/privkey.pem ] || @@ -278,6 +285,39 @@ if [ -f "$MANIFEST" ]; then IFS=$old_ifs fi +host_otel_config="$ROOT/deployment/observability/otel-host-collector.yaml" +if [ -f "$host_otel_config" ]; then + grep -Fq 'filelog/docker' "$host_otel_config" || + fail "host collector must include Docker filelog" + grep -Fq 'journald/host' "$host_otel_config" || + fail "host collector must include journald allow-list" + grep -Fq 'filter/allowlist' "$host_otel_config" || + fail "host collector must drop non-allowlisted containers" + grep -Fq 'file_storage' "$host_otel_config" || + fail "host collector must persist offsets and exporter queue" + ! grep -Fq '/var/run/docker.sock' "$host_otel_config" || + fail "Docker socket access is forbidden" +fi +[ -x /usr/local/bin/otelcol-contrib ] || + fail "pinned host otelcol-contrib is not installed" +otel_checksum=/usr/local/share/han-otel/otelcol-contrib.sha256 +otel_version=/usr/local/share/han-otel/otelcol-contrib.version +[ -f "$otel_checksum" ] || fail "host Collector checksum record is missing" +[ -f "$otel_version" ] || fail "host Collector version record is missing" +if [ -f "$otel_checksum" ]; then + (cd / && sha256sum -c "$otel_checksum") || + fail "host Collector binary checksum mismatch" +fi +if [ -x /usr/local/bin/otelcol-contrib ] && [ -f "$otel_version" ]; then + /usr/local/bin/otelcol-contrib --version 2>&1 | + grep -Fq "$(cat "$otel_version")" || + fail "host Collector binary version mismatch" +fi +if [ -x /usr/local/bin/otelcol-contrib ] && [ -f "$host_otel_config" ]; then + /usr/local/bin/otelcol-contrib validate --config="$host_otel_config" || + fail "host collector config validation failed" +fi + if [ -x "$ROOT/scripts/validate-env" ] && [ -f "$ENV_FILE" ] && [ -f "$MANIFEST" ]; then "$ROOT/scripts/validate-env" "$ENV_FILE" --runtime-manifest "$MANIFEST" || fail "config/runtime validator rejected the production inputs" diff --git a/VM1_app/codebase/backend/deployment/scripts/setup-vm.sh b/VM1_app/codebase/backend/deployment/scripts/setup-vm.sh index a4f94e6..81f6313 100644 --- a/VM1_app/codebase/backend/deployment/scripts/setup-vm.sh +++ b/VM1_app/codebase/backend/deployment/scripts/setup-vm.sh @@ -27,6 +27,8 @@ HARDEN_SSH="${HARDEN_SSH:-false}" LOCK_ACCOUNT_PASSWORDS="${LOCK_ACCOUNT_PASSWORDS:-true}" RESET_UFW="${RESET_UFW:-true}" SKIP_APT_UPGRADE="${SKIP_APT_UPGRADE:-false}" +OTEL_HOST_COLLECTOR_VERSION="${OTEL_HOST_COLLECTOR_VERSION:-}" +OTEL_HOST_COLLECTOR_SHA256="${OTEL_HOST_COLLECTOR_SHA256:-}" LOG_FILE="${LOG_FILE:-/var/log/han-chat-vm1-setup.log}" log() { @@ -120,6 +122,13 @@ configure_time() { timedatectl set-ntp true } +configure_journald() { + step "Persistent journald для host telemetry" + install -d -m 2755 -o root -g systemd-journal /var/log/journal + systemd-tmpfiles --create --prefix /var/log/journal + journalctl --flush +} + install_authorized_key() { local user=$1 source=$2 target="/home/${1}/.ssh/authorized_keys" [[ -f "$source" && ! -L "$source" ]] || die "Не найден обычный key file ${source}" @@ -422,7 +431,7 @@ install_deploy_sudoers() { step "Exact sudoers для deploy" cat >/etc/sudoers.d/deploy <<'EOF' Cmnd_Alias HAN_VM1_UNITS = /usr/bin/systemctl start han-secrets@production.service, /usr/bin/systemctl restart han-secrets@production.service, /usr/bin/systemctl start han-stack@production.service, /usr/bin/systemctl restart han-stack@production.service, /usr/bin/systemctl stop han-stack@production.service -Cmnd_Alias HAN_VM1_STATUS = /usr/bin/systemctl --no-pager status han-secrets@production.service, /usr/bin/systemctl --no-pager status han-stack@production.service, /usr/bin/journalctl --no-pager -u han-secrets@production.service, /usr/bin/journalctl --no-pager -u han-stack@production.service +Cmnd_Alias HAN_VM1_STATUS = /usr/bin/systemctl --no-pager status han-secrets@production.service, /usr/bin/systemctl --no-pager status han-stack@production.service, /usr/bin/systemctl --no-pager status han-host-otel-collector@production.service, /usr/bin/journalctl --no-pager -u han-secrets@production.service, /usr/bin/journalctl --no-pager -u han-stack@production.service, /usr/bin/journalctl --no-pager -u han-host-otel-collector@production.service deploy ALL=(root) NOPASSWD: HAN_VM1_UNITS, HAN_VM1_STATUS EOF chmod 0440 /etc/sudoers.d/deploy @@ -447,6 +456,15 @@ install_release_helpers_if_possible() { [[ -x "${deployment}/scripts/tls-deploy-hook.sh" ]] || die "TLS hook не executable" [[ -x "${source_dir}/han-compose" ]] || die "han-compose не executable" [[ -x "${source_dir}/han-secrets" ]] || die "han-secrets не executable" + [[ -f "${deployment}/han-host-otel-collector@.service" ]] \ + || die "han-host-otel-collector@.service отсутствует" + [[ -f "${deployment}/observability/otel-host-collector.yaml" ]] \ + || die "otel-host-collector.yaml отсутствует" + [[ "$OTEL_HOST_COLLECTOR_VERSION" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]] \ + || die "Задайте OTEL_HOST_COLLECTOR_VERSION=x.y.z" + [[ "$OTEL_HOST_COLLECTOR_SHA256" =~ ^[a-f0-9]{64}$ ]] \ + || die "Задайте проверенный OTEL_HOST_COLLECTOR_SHA256" + [[ "$(uname -m)" == x86_64 ]] || die "Host Collector artifact рассчитан на amd64" if getent group "$tls_group" >/dev/null; then [[ "$(getent group "$tls_group" | cut -d: -f3)" == "$tls_gid" ]] \ || die "han-nginx-tls имеет неожиданный GID" @@ -472,6 +490,27 @@ install_release_helpers_if_possible() { install -m 0644 -o root -g root \ "${deployment}/han-stack@.service" \ /etc/systemd/system/han-stack@.service + local otel_archive otel_tmp otel_url + otel_archive="$(mktemp)" + otel_tmp="$(mktemp -d)" + otel_url="https://github.com/open-telemetry/opentelemetry-collector-releases/releases/download/v${OTEL_HOST_COLLECTOR_VERSION}/otelcol-contrib_${OTEL_HOST_COLLECTOR_VERSION}_linux_amd64.tar.gz" + curl --fail --location --proto '=https' --tlsv1.2 \ + --output "$otel_archive" "$otel_url" + printf '%s %s\n' "$OTEL_HOST_COLLECTOR_SHA256" "$otel_archive" | sha256sum -c - + tar -xzf "$otel_archive" -C "$otel_tmp" otelcol-contrib + install -m 0755 -o root -g root "$otel_tmp/otelcol-contrib" /usr/local/bin/otelcol-contrib + install -d -m 0755 -o root -g root /usr/local/share/han-otel + sha256sum /usr/local/bin/otelcol-contrib \ + >/usr/local/share/han-otel/otelcol-contrib.sha256 + printf '%s\n' "$OTEL_HOST_COLLECTOR_VERSION" \ + >/usr/local/share/han-otel/otelcol-contrib.version + chmod 0644 /usr/local/share/han-otel/otelcol-contrib.sha256 \ + /usr/local/share/han-otel/otelcol-contrib.version + rm -rf "$otel_archive" "$otel_tmp" + install -d -m 0700 -o root -g root /var/lib/han-otel/host-collector + install -m 0644 -o root -g root \ + "${deployment}/han-host-otel-collector@.service" \ + /etc/systemd/system/han-host-otel-collector@.service install -d -m 0755 -o root -g root /etc/letsencrypt/renewal-hooks/deploy install -m 0755 -o root -g root \ "${deployment}/scripts/tls-deploy-hook.sh" \ @@ -481,6 +520,7 @@ install_release_helpers_if_possible() { /usr/local/share/doc/han-secrets/SELECTEL_RUNBOOK.ru.md systemctl daemon-reload systemctl enable certbot.timer + systemctl enable han-host-otel-collector@production.service log "Helpers установлены; application units не запущены" } @@ -539,6 +579,7 @@ main() { check_os update_system configure_time + configure_journald create_host_roles configure_account_passwords configure_layout diff --git a/VM1_app/codebase/backend/deployment/scripts/verify-observability.sh b/VM1_app/codebase/backend/deployment/scripts/verify-observability.sh index 9d591be..01bee9d 100644 --- a/VM1_app/codebase/backend/deployment/scripts/verify-observability.sh +++ b/VM1_app/codebase/backend/deployment/scripts/verify-observability.sh @@ -16,11 +16,38 @@ compose() { docker compose --env-file "$CONFIG_FILE" "$@"; } NETWORK="${OBSERVABILITY_NETWORK:-han-chat-observability}" COLLECTOR_SERVICE="${COLLECTOR_SERVICE:-otel-collector}" +TELEMETRYGEN_IMAGE="${TELEMETRYGEN_IMAGE:-}" errors=0 ok() { printf 'OK %s\n' "$*"; } fail() { printf 'FAIL %s\n' "$*" >&2; errors=$((errors + 1)); } +if systemctl is-active --quiet han-host-otel-collector@production.service; then + ok "Host Collector service is running" +else + fail "Host Collector service is not running" +fi + +if /usr/local/bin/otelcol-contrib validate \ + --config=deployment/observability/otel-host-collector.yaml >/dev/null 2>&1; then + ok "Host Collector configuration is valid" +else + fail "Host Collector configuration is invalid" +fi + +host_bad_logs="$( + journalctl --since=-10min --no-pager \ + -u han-host-otel-collector@production.service 2>&1 | + grep -Ei 'queue is full|permission denied|connection refused|tls:|Unauthenticated|Permanent error' || + true +)" +if [[ -z "$host_bad_logs" ]]; then + ok "No host Collector errors in last 10 minutes" +else + fail "Host Collector reports read/export/queue errors" + printf '%s\n' "$host_bad_logs" >&2 +fi + collector_id="$(compose ps -q "$COLLECTOR_SERVICE" 2>/dev/null || true)" if [[ -n "$collector_id" ]] && [[ "$(docker inspect --format '{{.State.Status}}' "$collector_id")" == running ]]; then @@ -43,12 +70,16 @@ PY fi done -docker run --rm --network "$NETWORK" \ - ghcr.io/open-telemetry/opentelemetry-collector-contrib/telemetrygen:latest \ +if [[ ! "$TELEMETRYGEN_IMAGE" =~ @sha256:[a-f0-9]{64}$ ]]; then + fail "TELEMETRYGEN_IMAGE must be pinned by sha256 digest" +elif docker run --rm --network "$NETWORK" \ + "$TELEMETRYGEN_IMAGE" \ traces --otlp-endpoint otel-collector:4317 --otlp-insecure \ - --service han-chat-e2e-canary --traces 100 --rate 20 >/dev/null \ - && ok "100 canary traces submitted" \ - || fail "telemetrygen failed" + --service han-chat-e2e-canary --traces 100 --rate 20 >/dev/null; then + ok "100 canary traces submitted" +else + fail "telemetrygen failed" +fi bad_logs="$( compose logs --since=10m "$COLLECTOR_SERVICE" 2>&1 | @@ -72,4 +103,12 @@ cat <<'EOF' service.namespace = han-chat Затем выполните synthetic API request и проверьте общий trace между api-backend и dependency spans, service.version и deployment.environment. +В Logs проверьте: + host.name = + service.name IN (nginx, keycloak, redis, api-backend, sms-service, + sms-worker, bitrix-local-app) +У canary application log trace_id/span_id должны открывать соответствующий +span. Python events должны встречаться один раз, unknown-container и +otel-host-collector отсутствовать. Fake token/PII marker не должен находиться +ни в logs, ни в traces. EOF diff --git a/VM1_app/codebase/backend/infra/compose/application.yml b/VM1_app/codebase/backend/infra/compose/application.yml index 61f3094..891dc3e 100644 --- a/VM1_app/codebase/backend/infra/compose/application.yml +++ b/VM1_app/codebase/backend/infra/compose/application.yml @@ -29,7 +29,7 @@ x-api-secrets: &api-secrets x-api-runtime: &api-runtime image: ${API_BACKEND_IMAGE:?API_BACKEND_IMAGE must be pinned by digest} - environment: + environment: &api-environment <<: *api-secret-environment APP_ENV: ${APP_ENV:-production-like} LOG_LEVEL: ${LOG_LEVEL:-INFO} @@ -167,6 +167,8 @@ services: KC_HTTP_RELATIVE_PATH: /auth KC_HEALTH_ENABLED: "true" KC_METRICS_ENABLED: "true" + KC_LOG_CONSOLE_OUTPUT: json + KC_LOG_LEVEL: ${KEYCLOAK_LOG_LEVEL:-info} KC_HOSTNAME: ${KEYCLOAK_PUBLIC_URL} PUBLIC_WEB_URL: ${PUBLIC_WEB_URL:?PUBLIC_WEB_URL is required for realm import} KC_BOOTSTRAP_ADMIN_USERNAME: ${KEYCLOAK_ADMIN} @@ -185,6 +187,8 @@ services: - keycloak_settings_bridge_token - keycloak_sms_service_token command: ["start", "--optimized", "--import-realm"] + labels: + com.han.service.name: keycloak expose: ["8080", "9000"] volumes: - type: bind @@ -209,7 +213,10 @@ services: core: {soft: 0, hard: 0} logging: driver: json-file - options: {max-size: "50m", max-file: "5"} + options: + max-size: "50m" + max-file: "5" + tag: keycloak sms-service: <<: *sms-runtime @@ -269,6 +276,9 @@ services: delivery-worker: <<: *api-runtime command: ["han-delivery-worker"] + environment: + <<: *api-environment + OTEL_SERVICE_NAME: delivery-worker depends_on: api-backend: {condition: service_healthy} bitrix-local-app: {condition: service_healthy} @@ -283,6 +293,9 @@ services: safety-recovery-worker: <<: *api-runtime command: ["han-safety-worker"] + environment: + <<: *api-environment + OTEL_SERVICE_NAME: safety-recovery-worker depends_on: api-backend: {condition: service_healthy} healthcheck: @@ -296,6 +309,9 @@ services: cleanup-worker: <<: *api-runtime command: ["han-cleanup-worker"] + environment: + <<: *api-environment + OTEL_SERVICE_NAME: cleanup-worker depends_on: api-backend: {condition: service_healthy} healthcheck: @@ -309,6 +325,9 @@ services: notification-expire-worker: <<: *api-runtime command: ["han-notification-expire-worker"] + environment: + <<: *api-environment + OTEL_SERVICE_NAME: notification-expire-worker depends_on: api-backend: {condition: service_healthy} healthcheck: @@ -322,6 +341,9 @@ services: notification-draft-cleanup-worker: <<: *api-runtime command: ["han-notification-draft-cleanup-worker"] + environment: + <<: *api-environment + OTEL_SERVICE_NAME: notification-draft-cleanup-worker depends_on: api-backend: {condition: service_healthy} healthcheck: @@ -346,6 +368,10 @@ services: BITRIX_API_FORWARD_TOKEN_FILE: /run/secrets/bitrix_api_forward_token BITRIX_TOKEN_ENCRYPTION_KEY_FILE: /run/secrets/bitrix_token_encryption_key APP_ENV: ${APP_ENV:-production-like} + LOG_LEVEL: ${LOG_LEVEL:-INFO} + RELEASE_VERSION: ${RELEASE_VERSION:-unknown} + OTEL_EXPORTER_OTLP_ENDPOINT: ${OTEL_EXPORTER_OTLP_ENDPOINT:-http://otel-collector:4317} + OTEL_SERVICE_NAME: bitrix-local-app BITRIX_CLIENT_ID: ${BITRIX_CLIENT_ID} BITRIX_CONNECTOR_ID: ${BITRIX_CONNECTOR_ID:-han_mobile_app} BITRIX_CONNECTOR_NAME: ${BITRIX_CONNECTOR_NAME:-HAN Mobile App} diff --git a/VM1_app/codebase/backend/nginx/docker-compose.yml b/VM1_app/codebase/backend/nginx/docker-compose.yml index 8ffd224..2877ad7 100644 --- a/VM1_app/codebase/backend/nginx/docker-compose.yml +++ b/VM1_app/codebase/backend/nginx/docker-compose.yml @@ -1,6 +1,8 @@ services: nginx: image: ${NGINX_IMAGE:?NGINX_IMAGE must be pinned by digest} + labels: + com.han.service.name: nginx environment: APP_ENV: ${APP_ENV:-production-like} PUBLIC_HOST: ${PUBLIC_HOST} @@ -64,4 +66,7 @@ services: nofile: {soft: 65536, hard: 65536} logging: driver: json-file - options: {max-size: "50m", max-file: "5"} + options: + max-size: "50m" + max-file: "5" + tag: nginx diff --git a/VM1_app/codebase/backend/observability/docker-compose.yml b/VM1_app/codebase/backend/observability/docker-compose.yml index d4670f1..85b2974 100644 --- a/VM1_app/codebase/backend/observability/docker-compose.yml +++ b/VM1_app/codebase/backend/observability/docker-compose.yml @@ -21,6 +21,7 @@ services: environment: APP_ENV: ${APP_ENV:-production-like} RELEASE_VERSION: ${RELEASE_VERSION:-unknown} + HOST_NAME: ${HOST_NAME:?HOST_NAME is required} OTEL_REMOTE_ENDPOINT: ${OTEL_REMOTE_ENDPOINT} OTEL_REMOTE_TLS_INSECURE: ${OTEL_REMOTE_TLS_INSECURE:-false} expose: ["4317", "4318", "13133", "8888"] diff --git a/VM1_app/codebase/backend/observability/otel-collector.yaml b/VM1_app/codebase/backend/observability/otel-collector.yaml index 3732d65..1d8a557 100644 --- a/VM1_app/codebase/backend/observability/otel-collector.yaml +++ b/VM1_app/codebase/backend/observability/otel-collector.yaml @@ -54,6 +54,7 @@ processors: - {key: service.namespace, value: han-chat, action: upsert} - {key: deployment.environment, value: "${env:APP_ENV}", action: upsert} - {key: service.version, value: "${env:RELEASE_VERSION}", action: upsert} + - {key: host.name, value: "${env:HOST_NAME}", action: upsert} attributes/redact: actions: - {key: http.request.header.authorization, action: delete} diff --git a/VM1_app/codebase/backend/redis/docker-compose.yml b/VM1_app/codebase/backend/redis/docker-compose.yml index 01d87e7..27e5b4f 100644 --- a/VM1_app/codebase/backend/redis/docker-compose.yml +++ b/VM1_app/codebase/backend/redis/docker-compose.yml @@ -1,6 +1,8 @@ services: redis: image: ${REDIS_IMAGE:?REDIS_IMAGE must be pinned by digest} + labels: + com.han.service.name: redis environment: REDIS_MAXMEMORY: ${REDIS_MAXMEMORY:-384mb} REDIS_EVICTION_POLICY: ${REDIS_EVICTION_POLICY:-volatile-lru} @@ -32,7 +34,10 @@ services: nofile: {soft: 65536, hard: 65536} logging: driver: json-file - options: {max-size: "50m", max-file: "5"} + options: + max-size: "50m" + max-file: "5" + tag: redis secrets: redis_api_password: diff --git a/VM1_app/codebase/backend/scripts/validate-env b/VM1_app/codebase/backend/scripts/validate-env index 0b0f253..b284a5f 100644 --- a/VM1_app/codebase/backend/scripts/validate-env +++ b/VM1_app/codebase/backend/scripts/validate-env @@ -13,12 +13,13 @@ from urllib.parse import urlparse REQUIRED_CONFIG = { "SECRETS_SOURCE", "APP_ENV", "RELEASE_VERSION", "HAN_PG_HOST", "HAN_PG_PORT", "HAN_PG_DATABASE", "PG_CA_HOST_PATH", "KEYCLOAK_DB_URL", "KEYCLOAK_DB_USERNAME", - "PUBLIC_HOST", "PUBLIC_WEB_URL", + "PUBLIC_HOST", "PUBLIC_WEB_URL", "HOST_NAME", "PUBLIC_API_URL", "PUBLIC_AUTH_URL", "KEYCLOAK_PUBLIC_URL", "KEYCLOAK_INTERNAL_URL", "KEYCLOAK_REALM", "KEYCLOAK_SMS_SERVICE_URL", "IDGTL_SMS_BASE_URL", "IDGTL_SMS_CALLBACK_PUBLIC_URL", "MESSAGE_SAFETY_URL", "MESSAGE_SAFETY_EXTRA_HOST", "MESSAGE_SAFETY_CA_HOST_PATH", "MESSAGE_SAFETY_API_PREFIX", + "OTEL_REMOTE_ENDPOINT", "OTEL_REMOTE_TLS_INSECURE", "TELEMETRYGEN_IMAGE", } REQUIRED_RUNTIME = { "DATABASE_URL", "BITRIX_DATABASE_URL", diff --git a/VM1_app/codebase/backend/sms-service/app/logging_security.py b/VM1_app/codebase/backend/sms-service/app/logging_security.py index db055fb..e19c686 100644 --- a/VM1_app/codebase/backend/sms-service/app/logging_security.py +++ b/VM1_app/codebase/backend/sms-service/app/logging_security.py @@ -1,7 +1,7 @@ from __future__ import annotations import re -from collections.abc import Mapping +from collections.abc import Mapping, MutableMapping from typing import Any REDACTED = "[REDACTED]" @@ -42,6 +42,6 @@ def sanitize_value(value: Any) -> Any: def redact_event( _logger: Any, _method_name: str, - event_dict: dict[str, Any], -) -> dict[str, Any]: + event_dict: MutableMapping[str, Any], +) -> MutableMapping[str, Any]: return sanitize_value(event_dict) diff --git a/VM1_app/codebase/backend/sms-service/app/main.py b/VM1_app/codebase/backend/sms-service/app/main.py index b070b5d..f86d81d 100644 --- a/VM1_app/codebase/backend/sms-service/app/main.py +++ b/VM1_app/codebase/backend/sms-service/app/main.py @@ -32,13 +32,18 @@ from app.service import ( read_message, ) from app.settings import get_settings -from app.telemetry import add_trace_context, init_telemetry, instrument_fastapi +from app.telemetry import TelemetryRuntime, add_trace_context, init_telemetry, instrument_fastapi log = structlog.get_logger() -def configure_logging(level: str) -> None: +def configure_logging(level: str, telemetry: TelemetryRuntime | None = None) -> None: logging.basicConfig(level=level, format="%(message)s") + if telemetry and telemetry.logging_handler not in logging.getLogger().handlers: + telemetry.logging_handler.addFilter( + lambda record: not record.name.startswith("opentelemetry") + ) + logging.getLogger().addHandler(telemetry.logging_handler) structlog.configure( processors=[ structlog.contextvars.merge_contextvars, @@ -47,7 +52,10 @@ def configure_logging(level: str) -> None: structlog.processors.TimeStamper(fmt="iso", utc=True, key="timestamp"), structlog.stdlib.add_log_level, structlog.processors.JSONRenderer(), - ] + ], + logger_factory=structlog.stdlib.LoggerFactory(), + wrapper_class=structlog.stdlib.BoundLogger, + cache_logger_on_first_use=True, ) @@ -55,7 +63,7 @@ def configure_logging(level: str) -> None: async def lifespan(app: FastAPI): settings = get_settings() telemetry = init_telemetry() - configure_logging(settings.log_level) + configure_logging(settings.log_level, telemetry) app.state.settings = settings app.state.db = Database(settings.database_url) try: diff --git a/VM1_app/codebase/backend/sms-service/app/telemetry.py b/VM1_app/codebase/backend/sms-service/app/telemetry.py index fbc0bde..0dcfb96 100644 --- a/VM1_app/codebase/backend/sms-service/app/telemetry.py +++ b/VM1_app/codebase/backend/sms-service/app/telemetry.py @@ -1,17 +1,23 @@ from __future__ import annotations +import logging import os +from collections.abc import MutableMapping from dataclasses import dataclass from typing import Any from fastapi import FastAPI from opentelemetry import metrics, trace +from opentelemetry._logs import set_logger_provider +from opentelemetry.exporter.otlp.proto.grpc._log_exporter import OTLPLogExporter from opentelemetry.exporter.otlp.proto.grpc.metric_exporter import OTLPMetricExporter from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter from opentelemetry.instrumentation.fastapi import FastAPIInstrumentor from opentelemetry.instrumentation.httpx import HTTPXClientInstrumentor from opentelemetry.instrumentation.sqlalchemy import SQLAlchemyInstrumentor from opentelemetry.propagate import set_global_textmap +from opentelemetry.sdk._logs import LoggerProvider, LoggingHandler +from opentelemetry.sdk._logs.export import BatchLogRecordProcessor from opentelemetry.sdk.metrics import MeterProvider from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader from opentelemetry.sdk.resources import Resource @@ -25,8 +31,11 @@ from opentelemetry.trace.propagation.tracecontext import TraceContextTextMapProp class TelemetryRuntime: tracer_provider: TracerProvider meter_provider: MeterProvider + logger_provider: LoggerProvider + logging_handler: LoggingHandler def shutdown(self) -> None: + self.logger_provider.shutdown() self.meter_provider.shutdown() self.tracer_provider.shutdown() @@ -53,7 +62,9 @@ def init_telemetry(service_name: str | None = None) -> TelemetryRuntime | None: if not endpoint: return None - resource = _resource(service_name or os.getenv("OTEL_SERVICE_NAME", "sms-service")) + resource = _resource( + service_name or os.getenv("OTEL_SERVICE_NAME", "sms-service") or "sms-service" + ) insecure = endpoint.startswith("http://") set_global_textmap(TraceContextTextMapPropagator()) @@ -77,9 +88,27 @@ def init_telemetry(service_name: str | None = None) -> TelemetryRuntime | None: meter_provider = MeterProvider(resource=resource, metric_readers=[metric_reader]) metrics.set_meter_provider(meter_provider) + logger_provider = LoggerProvider(resource=resource) + logger_provider.add_log_record_processor( + BatchLogRecordProcessor( + OTLPLogExporter(endpoint=endpoint, insecure=insecure, timeout=3), + max_queue_size=2048, + schedule_delay_millis=5000, + max_export_batch_size=512, + export_timeout_millis=3000, + ) + ) + set_logger_provider(logger_provider) + logging_handler = LoggingHandler(level=logging.NOTSET, logger_provider=logger_provider) + HTTPXClientInstrumentor().instrument() SQLAlchemyInstrumentor().instrument(enable_commenter=False) - _runtime = TelemetryRuntime(tracer_provider, meter_provider) + _runtime = TelemetryRuntime( + tracer_provider, + meter_provider, + logger_provider, + logging_handler, + ) return _runtime @@ -93,8 +122,8 @@ def instrument_fastapi(app: FastAPI) -> None: def add_trace_context( _logger: Any, _method_name: str, - event_dict: dict[str, Any], -) -> dict[str, Any]: + event_dict: MutableMapping[str, Any], +) -> MutableMapping[str, Any]: context = trace.get_current_span().get_span_context() if context.is_valid: event_dict["trace_id"] = format(context.trace_id, "032x") diff --git a/VM1_app/codebase/backend/sms-service/app/worker.py b/VM1_app/codebase/backend/sms-service/app/worker.py index 043030c..12cc3b4 100644 --- a/VM1_app/codebase/backend/sms-service/app/worker.py +++ b/VM1_app/codebase/backend/sms-service/app/worker.py @@ -10,6 +10,7 @@ from datetime import UTC, datetime, timedelta import httpx import structlog from opentelemetry import trace +from opentelemetry.propagate import extract from prometheus_client import start_http_server from sqlalchemy import and_, func, or_, select, update @@ -26,14 +27,27 @@ from app.metrics import ( from app.provider import IdgtlClient, IdgtlConfig from app.service import RuntimeSettings, load_runtime_settings from app.settings import Settings, get_settings -from app.telemetry import add_trace_context, init_telemetry +from app.telemetry import TelemetryRuntime, add_trace_context, init_telemetry log = structlog.get_logger() MAX_CONNECT_ATTEMPTS = 3 -def configure_logging(level: str) -> None: +def origin_links(traceparent: str | None) -> list[trace.Link]: + if not traceparent: + return [] + context = extract({"traceparent": traceparent}) + span_context = trace.get_current_span(context).get_span_context() + return [trace.Link(span_context)] if span_context.is_valid else [] + + +def configure_logging(level: str, telemetry: TelemetryRuntime | None = None) -> None: logging.basicConfig(level=level, format="%(message)s") + if telemetry and telemetry.logging_handler not in logging.getLogger().handlers: + telemetry.logging_handler.addFilter( + lambda record: not record.name.startswith("opentelemetry") + ) + logging.getLogger().addHandler(telemetry.logging_handler) structlog.configure( processors=[ structlog.contextvars.merge_contextvars, @@ -42,7 +56,10 @@ def configure_logging(level: str) -> None: structlog.processors.TimeStamper(fmt="iso", utc=True, key="timestamp"), structlog.stdlib.add_log_level, structlog.processors.JSONRenderer(), - ] + ], + logger_factory=structlog.stdlib.LoggerFactory(), + wrapper_class=structlog.stdlib.BoundLogger, + cache_logger_on_first_use=True, ) @@ -177,7 +194,7 @@ async def update_queue_metrics(db: Database) -> None: async def worker_loop(stop: asyncio.Event) -> None: settings = get_settings() telemetry = init_telemetry("sms-worker") - configure_logging(settings.log_level) + configure_logging(settings.log_level, telemetry) structlog.contextvars.bind_contextvars(**{"service.name": "sms-worker"}) metrics_server, metrics_thread = start_http_server( settings.metrics_port, @@ -200,8 +217,17 @@ async def worker_loop(stop: asyncio.Event) -> None: await update_queue_metrics(db) await asyncio.wait_for(stop.wait(), timeout=runtime.poll_interval_ms / 1000) continue - process_span = tracer.start_span("sms.process", start_time=claim_started_ns) - with trace.use_span(process_span, end_on_exit=True): + process_span = tracer.start_span( + "sms.process", + start_time=claim_started_ns, + links=origin_links(message.traceparent), + ) + with ( + trace.use_span(process_span, end_on_exit=True), + structlog.contextvars.bound_contextvars( + sms_message_id=str(message.id) + ), + ): claim_span = tracer.start_span("sms.claim", start_time=claim_started_ns) claim_span.set_attribute("sms.claimed", True) claim_span.end(end_time=claim_finished_ns) diff --git a/VM1_app/codebase/backend/sms-service/tests/unit/test_telemetry.py b/VM1_app/codebase/backend/sms-service/tests/unit/test_telemetry.py index c597598..ef996d6 100644 --- a/VM1_app/codebase/backend/sms-service/tests/unit/test_telemetry.py +++ b/VM1_app/codebase/backend/sms-service/tests/unit/test_telemetry.py @@ -1,6 +1,7 @@ from opentelemetry.sdk.trace import TracerProvider from app import telemetry +from app.worker import origin_links def test_telemetry_is_fail_open_without_endpoint(monkeypatch) -> None: @@ -19,3 +20,8 @@ def test_structlog_processor_adds_active_trace_context() -> None: assert result["event"] == "sms.safe" assert len(result["trace_id"]) == 32 assert len(result["span_id"]) == 16 + + +def test_sms_worker_rejects_invalid_origin_context() -> None: + assert origin_links(None) == [] + assert origin_links("00-invalid") == [] diff --git a/VM1_app/codebase/backend/tests/test_config.py b/VM1_app/codebase/backend/tests/test_config.py index e3bd2a9..3d61a60 100644 --- a/VM1_app/codebase/backend/tests/test_config.py +++ b/VM1_app/codebase/backend/tests/test_config.py @@ -348,9 +348,63 @@ class InfrastructureConfigTests(unittest.TestCase): source = service_main.read_text(encoding="utf-8") self.assertNotIn("route=request.url.path", source) self.assertIn('getattr(request.scope.get("route"), "path"', source) + for telemetry_path in ( + ROOT / "api-backend/app/telemetry.py", + ROOT / "sms-service/app/telemetry.py", + ROOT / "bitrix-local-app/app/telemetry.py", + ): + telemetry = telemetry_path.read_text(encoding="utf-8") + self.assertIn("OTLPLogExporter", telemetry) + self.assertIn("BatchLogRecordProcessor", telemetry) + self.assertIn("LoggingHandler", telemetry) worker = (ROOT / "sms-service/app/worker.py").read_text(encoding="utf-8") for span_name in ("sms.claim", "sms.process", "sms.provider", "sms.save_result"): self.assertIn(f'"{span_name}"', worker) + bitrix = (ROOT / "bitrix-local-app/app/main.py").read_text(encoding="utf-8") + self.assertIn('EXPECTED_BITRIX_DB_REVISION = "0002_inbox_trace_context"', bitrix) + self.assertIn("links=origin_links(row.traceparent)", bitrix) + bitrix_migration = ( + ROOT / "bitrix-local-app/alembic/versions/0002_inbox_trace_context.py" + ).read_text(encoding="utf-8") + self.assertIn('down_revision: str | None = "0001_bitrix_local"', bitrix_migration) + + def test_host_otel_collector_is_hardened_and_allowlisted(self) -> None: + config = ( + ROOT / "deployment/observability/otel-host-collector.yaml" + ).read_text(encoding="utf-8") + unit = (ROOT / "deployment/han-host-otel-collector@.service").read_text( + encoding="utf-8" + ) + setup = (ROOT / "deployment/scripts/setup-vm.sh").read_text(encoding="utf-8") + for required in ( + "filelog/docker", + "journald/host", + "filter/allowlist", + "storage: file_storage", + 'layout: "2006-01-02T15:04:05.000000000Z07:00"', + 'set(body, "nginx.access")', + 'set(body, "keycloak.event")', + '{key: uri, action: delete}', + 'resource.attributes["service.name"] != "nginx"', + 'resource.attributes["service.name"] != "keycloak"', + 'resource.attributes["service.name"] != "redis"', + ): + self.assertIn(required, config) + self.assertNotIn("/var/run/docker.sock", config + unit) + self.assertNotIn('from: attributes["service.name"]', config) + self.assertIn("CapabilityBoundingSet=", unit) + self.assertIn("NoNewPrivileges=yes", unit) + self.assertIn("ReadOnlyPaths=/var/lib/docker/containers", unit) + self.assertIn("ReadWritePaths=/var/lib/han-otel/host-collector", unit) + self.assertIn("OTEL_HOST_COLLECTOR_SHA256", setup) + self.assertIn("sha256sum -c -", setup) + self.assertIn("journalctl --flush", setup) + self.assertNotIn("systemctl start han-host-otel-collector", setup) + verify = ( + ROOT / "deployment/scripts/verify-observability.sh" + ).read_text(encoding="utf-8") + self.assertIn("TELEMETRYGEN_IMAGE", verify) + self.assertNotIn("telemetrygen:latest", verify) def test_settings_cli_and_workers_are_deployable(self) -> None: cli = ROOT / "api-backend/app/cli" @@ -442,7 +496,7 @@ class InfrastructureConfigTests(unittest.TestCase): dedup_fix, ) self.assertIn('down_revision: str | None = "0009_chat_message_max"', dedup_fix) - self.assertIn('EXPECTED_API_DB_REVISION = "0012_safety_v2_checkpoint"', main) + self.assertIn('EXPECTED_API_DB_REVISION = "0013_delivery_trace_context"', main) def test_consent_audit_migration_supports_existing_and_fresh_databases(self) -> None: migration = ( diff --git a/VM1_app/documentation/module-09-observability-vm1.md b/VM1_app/documentation/module-09-observability-vm1.md index e16fe96..ad6a19d 100644 --- a/VM1_app/documentation/module-09-observability-vm1.md +++ b/VM1_app/documentation/module-09-observability-vm1.md @@ -22,8 +22,14 @@ | Keycloak | `keycloak` | | `sms-service` | `sms-service` | | `sms-worker` | `sms-worker` | +| delivery worker | `delivery-worker` | +| safety recovery worker | `safety-recovery-worker` | +| quarantine cleanup worker | `cleanup-worker` | +| notification expiration worker | `notification-expire-worker` | +| notification draft cleanup worker | `notification-draft-cleanup-worker` | | Redis DB0/DB1 | `redis` | | local Collector | `otel-collector` | +| host telemetry agent | `otel-host-collector` | SMS-метрики и алерты детализированы в [`module-11-idgtl-sms.md`](module-11-idgtl-sms.md); имена зарегистрированы в arch-07 §4. @@ -36,6 +42,22 @@ SMS-метрики и алерты детализированы в [`module-11-i - export в SigNoz `192.168.0.5:4317`; hostname collector ВМ2 не используется; - pipeline, processors, limits, `otel-queue-init` и fail-open — arch-07 §3, §13, §14. +Python-сервисы экспортируют structured logs через OTLP Logs SDK в Compose +Collector. JSON stdout сохраняется только как bounded аварийный buffer и не +собирается повторно. + +Отдельный hardened host agent собирает по allow-list: + +- Docker `json-file` для nginx, Keycloak и Redis; +- journal units `docker`, `han-stack`, `han-secrets`, Docker firewall, + certbot и fail2ban; +- UFW log, если он включён на host. + +Host agent не имеет доступа к Docker socket, исключает собственные логи и +Compose Collector, хранит offsets/queue в `/var/lib/han-otel/host-collector` +и экспортирует напрямую в тот же SigNoz. Platform events без активного span +не получают искусственные trace IDs и ищутся по host/service/time window. + Scrape targets ВМ1 (кроме самого Collector): Keycloak management metrics, Redis exporter приложения, nginx exporter, сервисные `/metrics` `api-backend` и `bitrix-local-app`, если они не идут OTLP. ## 4. Instrumentation diff --git a/VM4_Expo-mobile/.assets/icons/icon.png b/VM4_Expo-mobile/.assets/icons/icon.png index 5e6227d..9c91540 100644 Binary files a/VM4_Expo-mobile/.assets/icons/icon.png and b/VM4_Expo-mobile/.assets/icons/icon.png differ diff --git a/VM4_Expo-mobile/app.config.ts b/VM4_Expo-mobile/app.config.ts index d4b0404..6b4b9eb 100644 --- a/VM4_Expo-mobile/app.config.ts +++ b/VM4_Expo-mobile/app.config.ts @@ -13,6 +13,9 @@ const config: ExpoConfig = { package: "ru.han.chat", versionCode: 2, softwareKeyboardLayoutMode: "resize", + // Expo prebuild template injects this into every AAB. The app does not draw + // over other windows; RuStore/Play treat it as a sensitive overlay permission. + blockedPermissions: ["android.permission.SYSTEM_ALERT_WINDOW"], adaptiveIcon: { foregroundImage: "./.assets/icons/adaptive-foreground.png", monochromeImage: "./.assets/icons/adaptive-monochrome.png", diff --git a/architectory/arch-07-observability.md b/architectory/arch-07-observability.md index 0534720..36887ea 100644 --- a/architectory/arch-07-observability.md +++ b/architectory/arch-07-observability.md @@ -184,12 +184,35 @@ service: - `deployment.environment=production-like|production`; - `host.name`/`service.instance.id` без публичного IP. +Для logs действуют те же resource attributes, что для traces/metrics. +`trace_id` (32 hex) и `span_id` (16 hex) добавляются только из активного +OpenTelemetry span. SigNoz связывает log со span по этой паре. Host, Redis, +Docker и systemd события, у которых span отсутствует, коррелируются по +`host.name`, `service.name`, времени и безопасным request/business IDs; +синтетические trace/span IDs для них запрещены. + +На application VM используются два непересекающихся канала: + +- Python application logs отправляются OTLP Logs SDK в Compose Collector и + одновременно остаются в JSON stdout как локальный аварийный buffer; +- platform logs (nginx, Keycloak, Redis, Docker и выбранные systemd units) + собирает отдельный host agent без доступа к Docker socket. + +Один источник не может одновременно экспортироваться SDK и host `filelog`. +Allow-list источников и self-exclusion Collector обязательны, чтобы исключить +дубли и feedback loop. + Реестр `service.name`: | Имя | Владелец | |---|---| | `nginx` | nginx каждой VM; различать экземпляры `host.name` / `service.instance.id` | | `api-backend` | ВМ1 | +| `delivery-worker` | ВМ1 | +| `safety-recovery-worker` | ВМ1 | +| `cleanup-worker` | ВМ1 | +| `notification-expire-worker` | ВМ1 | +| `notification-draft-cleanup-worker` | ВМ1 | | `bitrix-local-app` | ВМ1 | | `keycloak` | ВМ1 | | `sms-service` | ВМ1, callback/internal API | @@ -198,6 +221,7 @@ service: | `bitrix-sync` | ВМ2 | | `redis` | Redis каждой VM; различать экземпляры | | `otel-collector` | collector каждой VM | +| `otel-host-collector` | host telemetry agent ВМ1 | Новое имя добавляется только сюда, затем в спецификацию владельца. diff --git a/support¬es/deploy_stories/260903_2.md b/support¬es/deploy_stories/260903_2.md index 23d60c0..62cf7a6 100644 --- a/support¬es/deploy_stories/260903_2.md +++ b/support¬es/deploy_stories/260903_2.md @@ -1,5 +1,9 @@ Поддержка версионирования приложений +# update images +API_BACKEND_IMAGE=cr.selcloud.ru/han-images/han-api-backend@sha256:ee1e1b94eb944c44eb60534aeb003f36636b1e9a3df4fbf82d5b1a3a955a5e34 +NGINX_IMAGE=cr.selcloud.ru/han-images/nginx@sha256:c64e5a6296ca183538306fef9ae49a3168637bd8a4c925884016da341691673f + # Копируем файл на ВМ1 (HAN_chat_specification\VM1_app\codebase\backend\deployment\app-settings.production-like.yaml) cd C:/Users/MI/Documents/Assistent/HAN_chat_specification/VM1_app/ @@ -38,4 +42,7 @@ editor /etc/han/vm1.env /usr/local/sbin/han-vm1-compose ps --format "table {{.Service}}\t{{.Status}}\t{{.Ports}}" -# Сборка тестовой версии приложения \ No newline at end of file +# Сборка тестовой версии приложения +cd C:\Users\MI\Documents\Assistent\HAN_chat_specification\VM4_Expo-mobile +npx eas-cli build --profile preview-rustore --platform android +npx eas-cli build --profile rustore --platform android diff --git a/support¬es/deploy_stories/260904_1.md b/support¬es/deploy_stories/260904_1.md new file mode 100644 index 0000000..3ee2fcb --- /dev/null +++ b/support¬es/deploy_stories/260904_1.md @@ -0,0 +1,127 @@ +Логирование операций + +1. env: + +изменение Release_Version + + две переменные: +HOST_NAME=devhanapp \ prodhanapp.ihan.ru. +TELEMETRYGEN_IMAGE=ghcr.io/open-telemetry/opentelemetry-collector-contrib/telemetrygen@sha256:9461e2c7213219467e5f775ea8652d15c990afb4d1ca94e62344f2025a7587d1 + +2. Обновить образы: + +API_BACKEND_IMAGE=cr.selcloud.ru/han-images/han-api-backend@sha256:5ec8336ce56de41880398f68a2174530ba4dde9cd8ae6151c42f1631ab29bc35 +SMS_SERVICE_IMAGE=cr.selcloud.ru/han-images/sms-service@sha256:404685c45b7391a78f95c014ce8fdff64a84ccd7ae5b280e7a65b524c109d898 +BITRIX_LOCAL_APP_IMAGE=cr.selcloud.ru/han-images/bitrix-local-app@sha256:442889dc20e76f2a16d093adb4ae3ae38508535bafda82654755d1da50693183 +NGINX_IMAGE=cr.selcloud.ru/han-images/nginx@sha256:2cd7d32967b400c15bb6aaa18e97e99d429b2b6f2bfd54b965fc7d42a91a228e + +3. Скопировать проект: + +```powershell +$Release = "1.0.1" +tar --exclude=backend/.env ` + --exclude='backend/**/__pycache__' ` + --exclude='backend/**/.pytest_cache' ` + --exclude='backend/**/.ruff_cache' ` + -czf "vm1-backend-$Release.tar.gz" ` + -C C:/Users/MI/Documents/Assistent/HAN_chat_specification/VM1_app/codebase backend +Get-FileHash "vm1-backend-$Release.tar.gz" -Algorithm SHA256 +scp "vm1-backend-$Release.tar.gz" devVM1Deploy:/var/lib/han-deploy/incoming/ +``` + +RELEASE='1.0.1' +EXPECTED_SHA256='900C7AF2AEC061CFA112F82F8511A8A21F43EDB356EA3EF8C3205F49B6F1DF8F' +ARCHIVE="/var/lib/han-deploy/incoming/vm1-backend-${RELEASE}.tar.gz" +printf '%s %s/n' "$EXPECTED_SHA256" "$ARCHIVE" | sha256sum --check - +tar -tvzf "$ARCHIVE" +if tar -tzf "$ARCHIVE" | grep -Eq '(^/|(^|/)\.\.(/|$)|^backend/\.env$)'; then + echo 'unsafe path or .env' >&2; exit 1 +fi +if tar -tzf "$ARCHIVE" | grep -Ev '^backend(/|$)' | grep -q .; then + echo 'archive has files outside backend' >&2; exit 1 +fi +if tar -tvzf "$ARCHIVE" | awk '$1 ~ /^[lh]/ {found=1} END {exit !found}'; then + echo 'symlink/hardlink is forbidden' >&2; exit 1 +fi + +TARGET="/opt/han-chat/releases/${RELEASE}" +test ! -e "$TARGET" +install -d -m 0755 -o root -g root "$TARGET" +tar --extract --gzip --file "$ARCHIVE" --directory "$TARGET" \ + --no-same-owner --no-same-permissions +test -f "$TARGET/backend/docker-compose.yml" +chmod 0755 \ + "$TARGET/backend/deployment/scripts/setup-vm.sh" \ + "$TARGET/backend/deployment/preflight.sh" \ + "$TARGET/backend/deployment/scripts/tls-deploy-hook.sh" \ + "$TARGET/backend/deployment/secrets/han-compose" \ + "$TARGET/backend/deployment/secrets/han-secrets" +test -x "$TARGET/backend/deployment/scripts/setup-vm.sh" +test -x "$TARGET/backend/deployment/preflight.sh" +test -x "$TARGET/backend/deployment/scripts/tls-deploy-hook.sh" +test -x "$TARGET/backend/deployment/secrets/han-compose" +test -x "$TARGET/backend/deployment/secrets/han-secrets" +if find "$TARGET/backend" -type l -print -quit | grep -q .; then exit 1; fi +chown -R root:root "$TARGET" +chmod -R go-w "$TARGET" +ln -s "releases/${RELEASE}" /opt/han-chat/.current-new +mv -Tf /opt/han-chat/.current-new /opt/han-chat/current + +cd /opt/han-chat/current/backend/ +sudo apt update +sudo apt install -y dos2unix +find . -type f \( \ + -name '*.sh' -o -name '*.py' -o -name '*.service' -o -name '*.sudoers' -o -name 'han-compose' -o -name 'han-secrets' -o -name 'han-message-safety-mode' -o -name '*.yml' -o -name '*.conf*' -o -name 'preflight.sh' -o -name '*.yaml' -o -name '*.md' -o -name '*.toml' -o -name 'Dockerfile' -o -name '*.acl*' -o -name '*.tsx' -o -name '*.ts' -o -name '*.java' -o -name '*.js' -o -name '*.ps1' -o -name '*.xml' -o -name '*.dockerignore' -o -name '*.env*' -o -name '*.css' -o -name '*.ftl' -o -name '*.TAG' -o -name '*validate-env*' -o -name '*.properties' -o -name '*org.keycloak.*Factory' -o -name '*.html' \ +\) -exec dos2unix {} + +find -type f -exec file {} \; | grep -i 'CRLF' + + +sudo apt remove -y dos2unix +sudo apt purge -y dos2unix + + +/usr/local/sbin/han-vm1-compose --profile ops run --rm migrate-api alembic current +/usr/local/sbin/han-vm1-compose --profile ops run --rm migrate-bitrix-local alembic current + +/usr/local/sbin/han-vm1-compose --profile ops run --rm migrate-api +/usr/local/sbin/han-vm1-compose --profile ops run --rm migrate-bitrix-local + +Проверка применения миграции +/usr/local/sbin/han-vm1-compose --profile ops run --rm migrate-bitrix-local alembic current +/usr/local/sbin/han-vm1-compose --profile ops run --rm migrate-bitrix-local alembic heads + + +scp ` + C:/Users/MI/.ssh/devDeploy.pub ` + C:/Users/MI/.ssh/devAdmin.pub ` +devVM1Deploy:/var/lib/han-deploy/incoming/ + +install -m 0600 -o root -g root /var/lib/han-deploy/incoming/devDeploy.pub /root/bootstrap/deploy.pub +install -m 0600 -o root -g root /var/lib/han-deploy/incoming/devAdmin.pub /root/bootstrap/admin.pub + +rm -f /var/lib/han-deploy/incoming/devDeploy.pub /var/lib/han-deploy/incoming/devAdmin.pub + +cd /opt/han-chat/current/backend + +export OTEL_HOST_COLLECTOR_VERSION='0.117.0' +export OTEL_HOST_COLLECTOR_SHA256='90710c909a30fc3b89dd0e389c13f9e57ebbb87d6a0dc51fdde0bf03499a5f25' +DEPLOY_AUTHORIZED_KEY_FILE=/root/bootstrap/deploy.pub \ +ADMIN_AUTHORIZED_KEY_FILE=/root/bootstrap/admin.pub \ +HARDEN_SSH=true SKIP_APT_UPGRADE=true \ +/opt/han-chat/current/backend/deployment/scripts/setup-vm.sh + +/usr/local/bin/otelcol-contrib validate \ + --config=/opt/han-chat/current/backend/deployment/observability/otel-host-collector.yaml + +chmod 0755 /opt/han-chat/current/backend/scripts/validate-env +/opt/han-chat/current/backend/deployment/preflight.sh + +systemctl restart han-stack@production.service +если ошибка systemctl status han-stack@production.service +docker pull cr.selcloud.ru/han-images/nginx@sha256:2cd7d32967b400c15bb6aaa18e97e99d429b2b6f2bfd54b965fc7d42a91a228e +docker pull cr.selcloud.ru/han-images/sms-service@sha256:404685c45b7391a78f95c014ce8fdff64a84ccd7ae5b280e7a65b524c109d898 + +systemctl start han-host-otel-collector@production.service + +Canary test: CONFIG_FILE=/etc/han/vm1.env ./deployment/scripts/verify-observability.sh + +rm -f /root/bootstrap/deploy.pub /root/bootstrap/admin.pub \ No newline at end of file