ВМ1: реализован сбор логов

VM1_app/codebase/backend/.env.example

@@ -115,6 +115,7 @@ SELECTEL_S3_BUCKET_ATTACHMENTS=han-chat-attachments
 SELECTEL_S3_BUCKET_QUARANTINE=han-chat-quarantine
 
 OTEL_EXPORTER_OTLP_ENDPOINT=http://otel-collector:4317
+HOST_NAME=vm1-production
 OTEL_SERVICE_NAME_API=api-backend
 OTEL_SERVICE_NAME_SMS_API=sms-service
 OTEL_SERVICE_NAME_SMS_WORKER=sms-worker
@@ -126,6 +127,8 @@ OTEL_REMOTE_ENDPOINT=otlp.example.invalid:4317
 OTEL_REMOTE_TLS_INSECURE=false
 # SDK отправляет все spans локальному Collector; решение о хранении принимает tail_sampling.
 OTEL_TRACES_SAMPLER=always_on
+# Операторский canary image; обязателен digest из утверждённого release record.
+TELEMETRYGEN_IMAGE=ghcr.io/open-telemetry/opentelemetry-collector-contrib/telemetrygen@sha256:<64-hex>
 # OTEL_QUEUE_SIZE пока документирует целевую ёмкость, конфигурация Collector закреплена в YAML.
 OTEL_QUEUE_SIZE=10000
 

VM1_app/codebase/backend/api-backend/alembic/versions/0013_delivery_trace_context.py

@@ -0,0 +1,45 @@
+"""Persist W3C trace context for asynchronous delivery.
+
+Revision ID: 0013_delivery_trace_context
+Revises: 0012_safety_v2_checkpoint
+Create Date: 2026-09-03
+"""
+
+from collections.abc import Sequence
+
+from alembic import op
+
+revision: str = "0013_delivery_trace_context"
+down_revision: str | None = "0012_safety_v2_checkpoint"
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
+
+
+def upgrade() -> None:
+    op.execute(
+        """
+        ALTER TABLE han_app.delivery_outbox
+          ADD COLUMN IF NOT EXISTS traceparent varchar(55)
+        """
+    )
+    op.execute(
+        """
+        ALTER TABLE han_app.safety_tasks
+          ADD COLUMN IF NOT EXISTS traceparent varchar(55)
+        """
+    )
+
+
+def downgrade() -> None:
+    op.execute(
+        """
+        ALTER TABLE han_app.delivery_outbox
+          DROP COLUMN IF EXISTS traceparent
+        """
+    )
+    op.execute(
+        """
+        ALTER TABLE han_app.safety_tasks
+          DROP COLUMN IF EXISTS traceparent
+        """
+    )

VM1_app/codebase/backend/api-backend/app/db.py

@@ -206,6 +206,7 @@ class SafetyTask(Base):
     message_id: Mapped[uuid.UUID] = mapped_column(ForeignKey(f"{SCHEMA}.messages.id"), unique=True)
     attachment_id: Mapped[uuid.UUID | None] = mapped_column(UUID(as_uuid=True))
     quarantine_object_key: Mapped[str | None] = mapped_column(String(1024))
+    traceparent: Mapped[str | None] = mapped_column(String(55))
     status: Mapped[str] = mapped_column(String(16))
     processing_mode: Mapped[str | None] = mapped_column(String(16))
     config_version: Mapped[int | None] = mapped_column(BigInteger)
@@ -228,6 +229,7 @@ class DeliveryOutbox(Base):
     message_id: Mapped[uuid.UUID] = mapped_column(ForeignKey(f"{SCHEMA}.messages.id"), unique=True)
     external_chat_id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True))
     payload_json: Mapped[dict[str, Any]] = mapped_column(JSON)
+    traceparent: Mapped[str | None] = mapped_column(String(55))
     status: Mapped[str] = mapped_column(String(16), default="pending")
     attempt_count: Mapped[int] = mapped_column(Integer, default=0)
     next_attempt_at: Mapped[datetime] = mapped_column(DateTime(timezone=True))

VM1_app/codebase/backend/api-backend/app/logging_security.py

@@ -1,7 +1,7 @@
 from __future__ import annotations
 
 import re
-from collections.abc import Mapping
+from collections.abc import Mapping, MutableMapping
 from typing import Any
 
 REDACTED = "[REDACTED]"
@@ -42,6 +42,6 @@ def sanitize_value(value: Any) -> Any:
 def redact_event(
     _logger: Any,
     _method_name: str,
-    event_dict: dict[str, Any],
+    event_dict: MutableMapping[str, Any],
-) -> dict[str, Any]:
+) -> MutableMapping[str, Any]:
     return sanitize_value(event_dict)

VM1_app/codebase/backend/api-backend/app/main.py

@@ -89,11 +89,22 @@ from app.services import (
     start_session,
 )
 from app.settings import get_settings
-from app.telemetry import add_trace_context, current_trace_id, init_telemetry, instrument_fastapi
+from app.telemetry import (
+    TelemetryRuntime,
+    add_trace_context,
+    current_trace_id,
+    init_telemetry,
+    instrument_fastapi,
+)
 
 
-def configure_logging(level: str) -> None:
+def configure_logging(level: str, telemetry: TelemetryRuntime | None = None) -> None:
     logging.basicConfig(level=level, format="%(message)s")
+    if telemetry and telemetry.logging_handler not in logging.getLogger().handlers:
+        telemetry.logging_handler.addFilter(
+            lambda record: not record.name.startswith("opentelemetry")
+        )
+        logging.getLogger().addHandler(telemetry.logging_handler)
     structlog.configure(
         processors=[
             structlog.contextvars.merge_contextvars,
@@ -102,7 +113,10 @@ def configure_logging(level: str) -> None:
             structlog.processors.TimeStamper(fmt="iso", utc=True, key="timestamp"),
             structlog.stdlib.add_log_level,
             structlog.processors.JSONRenderer(),
-        ]
+        ],
+        logger_factory=structlog.stdlib.LoggerFactory(),
+        wrapper_class=structlog.stdlib.BoundLogger,
+        cache_logger_on_first_use=True,
     )
 
 
@@ -135,7 +149,7 @@ async def refresh_jwks_cache(app: FastAPI) -> None:
 async def lifespan(app: FastAPI):
     settings = get_settings()
     telemetry = init_telemetry()
-    configure_logging(settings.log_level)
+    configure_logging(settings.log_level, telemetry)
     app.state.settings = settings
     app.state.db = Database(settings.database_url)
     app.state.http = httpx.AsyncClient()
@@ -178,7 +192,7 @@ async def lifespan(app: FastAPI):
             telemetry.shutdown()
 
 
-EXPECTED_API_DB_REVISION = "0012_safety_v2_checkpoint"
+EXPECTED_API_DB_REVISION = "0013_delivery_trace_context"
 
 
 app = FastAPI(

VM1_app/codebase/backend/api-backend/app/services.py

@@ -58,6 +58,7 @@ from app.schemas import (
     encode_cursor,
 )
 from app.settings import Settings
+from app.telemetry import current_traceparent
 
 MESSAGE_SAFETY_REPLIES = {
     "text": (
@@ -95,6 +96,7 @@ async def ensure_delivery_outbox(
     external_chat_id: uuid.UUID,
     payload_json: dict[str, Any],
     next_attempt_at: datetime,
+    traceparent: str | None = None,
 ) -> DeliveryOutbox:
     """Create the per-message outbox row or return the concurrent winner."""
     statement = (
@@ -104,6 +106,7 @@ async def ensure_delivery_outbox(
             message_id=message_id,
             external_chat_id=external_chat_id,
             payload_json=payload_json,
+            traceparent=traceparent,
             next_attempt_at=next_attempt_at,
         )
         .on_conflict_do_nothing(index_elements=[DeliveryOutbox.message_id])
@@ -1039,6 +1042,7 @@ async def send_message(
                 message_id=message.id,
                 attachment_id=attachment.id if attachment else None,
                 quarantine_object_key=attachment.quarantine_object_key if attachment else None,
+                traceparent=current_traceparent(),
                 status="polling",
                 processing_mode=verdict["processing_mode"],
                 config_version=verdict["config_version"],
@@ -1133,6 +1137,7 @@ async def send_message(
             # delivery worker race the synchronous first attempt.
             next_attempt_at=datetime.now(UTC)
             + timedelta(seconds=settings.bitrix_local_app_http_timeout_sec + 5),
+            traceparent=current_traceparent(),
         )
         await session.commit()
         await publish_message_status(fanout, message, settings)

VM1_app/codebase/backend/api-backend/app/telemetry.py

@@ -1,11 +1,15 @@
 from __future__ import annotations
 
+import logging
 import os
+from collections.abc import MutableMapping
 from dataclasses import dataclass
 from typing import Any
 
 from fastapi import FastAPI
 from opentelemetry import metrics, trace
+from opentelemetry._logs import set_logger_provider
+from opentelemetry.exporter.otlp.proto.grpc._log_exporter import OTLPLogExporter
 from opentelemetry.exporter.otlp.proto.grpc.metric_exporter import OTLPMetricExporter
 from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter
 from opentelemetry.instrumentation.botocore import BotocoreInstrumentor
@@ -13,7 +17,9 @@ from opentelemetry.instrumentation.fastapi import FastAPIInstrumentor
 from opentelemetry.instrumentation.httpx import HTTPXClientInstrumentor
 from opentelemetry.instrumentation.redis import RedisInstrumentor
 from opentelemetry.instrumentation.sqlalchemy import SQLAlchemyInstrumentor
-from opentelemetry.propagate import set_global_textmap
+from opentelemetry.propagate import extract, inject, set_global_textmap
+from opentelemetry.sdk._logs import LoggerProvider, LoggingHandler
+from opentelemetry.sdk._logs.export import BatchLogRecordProcessor
 from opentelemetry.sdk.metrics import MeterProvider
 from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader
 from opentelemetry.sdk.resources import Resource
@@ -27,8 +33,11 @@ from opentelemetry.trace.propagation.tracecontext import TraceContextTextMapProp
 class TelemetryRuntime:
     tracer_provider: TracerProvider
     meter_provider: MeterProvider
+    logger_provider: LoggerProvider
+    logging_handler: LoggingHandler
 
     def shutdown(self) -> None:
+        self.logger_provider.shutdown()
         self.meter_provider.shutdown()
         self.tracer_provider.shutdown()
 
@@ -79,11 +88,29 @@ def init_telemetry(service_name: str | None = None) -> TelemetryRuntime | None:
     meter_provider = MeterProvider(resource=resource, metric_readers=[metric_reader])
     metrics.set_meter_provider(meter_provider)
 
+    logger_provider = LoggerProvider(resource=resource)
+    logger_provider.add_log_record_processor(
+        BatchLogRecordProcessor(
+            OTLPLogExporter(endpoint=endpoint, insecure=insecure, timeout=3),
+            max_queue_size=2048,
+            schedule_delay_millis=5000,
+            max_export_batch_size=512,
+            export_timeout_millis=3000,
+        )
+    )
+    set_logger_provider(logger_provider)
+    logging_handler = LoggingHandler(level=logging.NOTSET, logger_provider=logger_provider)
+
     HTTPXClientInstrumentor().instrument()
     SQLAlchemyInstrumentor().instrument(enable_commenter=False)
     RedisInstrumentor().instrument()
     BotocoreInstrumentor().instrument()
-    _runtime = TelemetryRuntime(tracer_provider, meter_provider)
+    _runtime = TelemetryRuntime(
+        tracer_provider,
+        meter_provider,
+        logger_provider,
+        logging_handler,
+    )
     return _runtime
 
 
@@ -97,8 +124,8 @@ def instrument_fastapi(app: FastAPI) -> None:
 def add_trace_context(
     _logger: Any,
     _method_name: str,
-    event_dict: dict[str, Any],
+    event_dict: MutableMapping[str, Any],
-) -> dict[str, Any]:
+) -> MutableMapping[str, Any]:
     context = trace.get_current_span().get_span_context()
     if context.is_valid:
         event_dict["trace_id"] = format(context.trace_id, "032x")
@@ -109,3 +136,17 @@ def add_trace_context(
 def current_trace_id() -> str | None:
     context = trace.get_current_span().get_span_context()
     return format(context.trace_id, "032x") if context.is_valid else None
+
+
+def current_traceparent() -> str | None:
+    carrier: dict[str, str] = {}
+    inject(carrier)
+    return carrier.get("traceparent")
+
+
+def origin_links(traceparent: str | None) -> list[trace.Link]:
+    if not traceparent:
+        return []
+    context = extract({"traceparent": traceparent})
+    span_context = trace.get_current_span(context).get_span_context()
+    return [trace.Link(span_context)] if span_context.is_valid else []

VM1_app/codebase/backend/api-backend/app/workers.py

@@ -1,12 +1,14 @@
 import asyncio
+import logging
 import uuid
-from collections.abc import AsyncIterator
+from collections.abc import AsyncIterator, Awaitable, Callable
 from contextlib import asynccontextmanager
 from datetime import UTC, datetime, timedelta
 
 import httpx
 import redis.asyncio as redis
 import structlog
+from opentelemetry import trace
 from sqlalchemy import delete, select
 
 from app.db import (
@@ -25,6 +27,7 @@ from app.integrations import (
     SafetyClient,
     fresh_openlines_payload,
 )
+from app.logging_security import redact_event
 from app.notification_models import ClientUploadDraft
 from app.notification_service import expire_notifications
 from app.realtime import RealtimeFanout
@@ -37,10 +40,45 @@ from app.services import (
     publish_message_status,
 )
 from app.settings import Settings, get_settings
+from app.telemetry import TelemetryRuntime, add_trace_context, init_telemetry, origin_links
 
 log = structlog.get_logger()
 
 
+def configure_logging(level: str, telemetry: TelemetryRuntime | None = None) -> None:
+    logging.basicConfig(level=level, format="%(message)s")
+    if telemetry and telemetry.logging_handler not in logging.getLogger().handlers:
+        telemetry.logging_handler.addFilter(
+            lambda record: not record.name.startswith("opentelemetry")
+        )
+        logging.getLogger().addHandler(telemetry.logging_handler)
+    structlog.configure(
+        processors=[
+            structlog.contextvars.merge_contextvars,
+            add_trace_context,
+            redact_event,
+            structlog.processors.TimeStamper(fmt="iso", utc=True, key="timestamp"),
+            structlog.stdlib.add_log_level,
+            structlog.processors.JSONRenderer(),
+        ],
+        logger_factory=structlog.stdlib.LoggerFactory(),
+        wrapper_class=structlog.stdlib.BoundLogger,
+        cache_logger_on_first_use=True,
+    )
+
+
+def run_worker(service_name: str, target: Callable[[], Awaitable[None]]) -> None:
+    settings = get_settings()
+    telemetry = init_telemetry(service_name)
+    configure_logging(settings.log_level, telemetry)
+    structlog.contextvars.bind_contextvars(**{"service.name": service_name})
+    try:
+        asyncio.run(target())
+    finally:
+        if telemetry:
+            telemetry.shutdown()
+
+
 @asynccontextmanager
 async def worker_http_clients(
     settings: Settings,
@@ -61,6 +99,7 @@ async def delivery_once(
     worker_id: str,
     batch_size: int = 20,
 ) -> int:
+    tracer = trace.get_tracer("han.api.delivery-worker")
     async with db.sessions() as session:
         rows = (
             (
@@ -88,6 +127,16 @@ async def delivery_once(
             row = await session.get(DeliveryOutbox, row_id, with_for_update=True)
             if row is None:
                 continue
+            with (
+                tracer.start_as_current_span(
+                    "delivery.process",
+                    links=origin_links(row.traceparent),
+                ),
+                structlog.contextvars.bound_contextvars(
+                    delivery_outbox_id=str(row.id),
+                    message_id=str(row.message_id),
+                ),
+            ):
                 message = None
                 dialog = None
                 try:
@@ -127,6 +176,7 @@ async def safety_once(
     worker_id: str,
     batch_size: int = 20,
 ) -> int:
+    tracer = trace.get_tracer("han.api.safety-recovery-worker")
     async with db.sessions() as session:
         rows = (
             (
@@ -155,7 +205,14 @@ async def safety_once(
                 continue
             message = None
             try:
-                verdict = await safety.poll(task.poll_location, f"worker-{worker_id}")
+                with tracer.start_as_current_span(
+                    "safety.recovery.poll",
+                    links=origin_links(task.traceparent),
+                ):
+                    verdict = await safety.poll(
+                        task.poll_location,
+                        f"worker-{worker_id}",
+                    )
                 message = await session.get(Message, task.message_id)
                 attachment = (
                     await session.get(MessageAttachment, task.attachment_id)
@@ -204,6 +261,7 @@ async def safety_once(
                                 attachment=attachment,
                             ),
                             next_attempt_at=datetime.now(UTC),
+                            traceparent=task.traceparent,
                         )
                 elif verdict["_status"] == 403 and message:
                     message.safety_processing_mode = verdict["processing_mode"]
@@ -365,20 +423,20 @@ async def notification_draft_cleanup_loop() -> None:
 
 
 def delivery_main() -> None:
-    asyncio.run(loop("delivery"))
+    run_worker("delivery-worker", lambda: loop("delivery"))
 
 
 def safety_main() -> None:
-    asyncio.run(loop("safety"))
+    run_worker("safety-recovery-worker", lambda: loop("safety"))
 
 
 def cleanup_main() -> None:
-    asyncio.run(loop("cleanup"))
+    run_worker("cleanup-worker", lambda: loop("cleanup"))
 
 
 def notification_expire_main() -> None:
-    asyncio.run(notification_expire_loop())
+    run_worker("notification-expire-worker", notification_expire_loop)
 
 
 def notification_draft_cleanup_main() -> None:
-    asyncio.run(notification_draft_cleanup_loop())
+    run_worker("notification-draft-cleanup-worker", notification_draft_cleanup_loop)

VM1_app/codebase/backend/api-backend/tests/unit/test_telemetry.py

@@ -23,3 +23,19 @@ def test_structlog_processor_adds_active_trace_context_only() -> None:
     assert len(result["trace_id"]) == 32
     assert len(result["span_id"]) == 16
     assert set(result) == {"event", "request_id", "trace_id", "span_id"}
+
+
+def test_no_active_span_does_not_invent_trace_context() -> None:
+    result = telemetry.add_trace_context(None, "info", {"event": "safe"})
+    assert result == {"event": "safe"}
+
+
+def test_async_origin_creates_link_without_parenting_worker_span() -> None:
+    tracer = TracerProvider().get_tracer("test")
+    with tracer.start_as_current_span("request"):
+        traceparent = telemetry.current_traceparent()
+        origin_trace_id = telemetry.current_trace_id()
+
+    links = telemetry.origin_links(traceparent)
+    assert len(links) == 1
+    assert format(links[0].context.trace_id, "032x") == origin_trace_id

VM1_app/codebase/backend/bitrix-local-app/alembic/versions/0002_inbox_trace_context.py

@@ -0,0 +1,63 @@
+"""Persist request and W3C trace context for inbox forwarding.
+
+Revision ID: 0002_inbox_trace_context
+Revises: 0001_bitrix_local_schema
+Create Date: 2026-09-03
+"""
+
+from collections.abc import Sequence
+
+from alembic import op
+
+revision: str = "0002_inbox_trace_context"
+down_revision: str | None = "0001_bitrix_local"
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
+
+
+def upgrade() -> None:
+    op.execute(
+        """
+        ALTER TABLE bitrix_local.inbox_events
+          ADD COLUMN IF NOT EXISTS request_id varchar(64),
+          ADD COLUMN IF NOT EXISTS traceparent varchar(55)
+        """
+    )
+    op.execute(
+        """
+        ALTER TABLE bitrix_local.outbound_messages
+          ADD COLUMN IF NOT EXISTS request_id varchar(64),
+          ADD COLUMN IF NOT EXISTS traceparent varchar(55)
+        """
+    )
+    op.execute(
+        """
+        ALTER TABLE bitrix_local.delivery_ack_outbox
+          ADD COLUMN IF NOT EXISTS request_id varchar(64),
+          ADD COLUMN IF NOT EXISTS traceparent varchar(55)
+        """
+    )
+
+
+def downgrade() -> None:
+    op.execute(
+        """
+        ALTER TABLE bitrix_local.inbox_events
+          DROP COLUMN IF EXISTS traceparent,
+          DROP COLUMN IF EXISTS request_id
+        """
+    )
+    op.execute(
+        """
+        ALTER TABLE bitrix_local.outbound_messages
+          DROP COLUMN IF EXISTS traceparent,
+          DROP COLUMN IF EXISTS request_id
+        """
+    )
+    op.execute(
+        """
+        ALTER TABLE bitrix_local.delivery_ack_outbox
+          DROP COLUMN IF EXISTS traceparent,
+          DROP COLUMN IF EXISTS request_id
+        """
+    )

VM1_app/codebase/backend/bitrix-local-app/app/logging_security.py

@@ -0,0 +1,47 @@
+from __future__ import annotations
+
+import re
+from collections.abc import Mapping, MutableMapping
+from typing import Any
+
+REDACTED = "[REDACTED]"
+_SENSITIVE_KEY = re.compile(
+    r"(authorization|cookie|password|passwd|secret|token|api[_-]?key|"
+    r"database[_-]?url|dsn|callback[_-]?url|phone|email|message|payload)",
+    re.IGNORECASE,
+)
+_URI_USERINFO = re.compile(r"(?P<scheme>[a-z][a-z0-9+.-]*://)[^/@\s]+@", re.IGNORECASE)
+_QUERY_SECRET = re.compile(
+    r"(?P<prefix>[?&](?:token|access_token|api_key|key|secret|password)=)[^&#\s]+",
+    re.IGNORECASE,
+)
+_AUTH_VALUE = re.compile(r"\b(Bearer|Basic)\s+[A-Za-z0-9._~+/=-]+", re.IGNORECASE)
+
+
+def sanitize_text(value: str) -> str:
+    value = _URI_USERINFO.sub(r"\g<scheme>[REDACTED]@", value)
+    value = _QUERY_SECRET.sub(r"\g<prefix>[REDACTED]", value)
+    return _AUTH_VALUE.sub(r"\1 [REDACTED]", value)
+
+
+def sanitize_value(value: Any) -> Any:
+    if isinstance(value, str):
+        return sanitize_text(value)
+    if isinstance(value, Mapping):
+        return {
+            str(key): REDACTED if _SENSITIVE_KEY.search(str(key)) else sanitize_value(item)
+            for key, item in value.items()
+        }
+    if isinstance(value, list):
+        return [sanitize_value(item) for item in value]
+    if isinstance(value, tuple):
+        return tuple(sanitize_value(item) for item in value)
+    return value
+
+
+def redact_event(
+    _logger: Any,
+    _method_name: str,
+    event_dict: MutableMapping[str, Any],
+) -> MutableMapping[str, Any]:
+    return sanitize_value(event_dict)

VM1_app/codebase/backend/bitrix-local-app/app/main.py

@@ -16,11 +16,13 @@ from typing import Annotated, Any, Literal
 from urllib.parse import quote, urlparse
 
 import httpx
+import structlog
 import uvicorn
 from cryptography.hazmat.primitives.ciphers.aead import AESGCM
 from fastapi import Depends, FastAPI, Header, HTTPException, Request
 from fastapi.exceptions import RequestValidationError
 from fastapi.responses import HTMLResponse, JSONResponse
+from opentelemetry import trace
 from pydantic import BaseModel, ConfigDict, Field, model_validator
 from pydantic_settings import BaseSettings, SettingsConfigDict
 from sqlalchemy import func, or_, select, text
@@ -37,9 +39,19 @@ from app.models import (
     PortalInstallation,
     now,
 )
+from app.logging_security import redact_event
 from app.postgres import create_postgres_engine
+from app.telemetry import (
+    TelemetryRuntime,
+    add_trace_context,
+    current_traceparent,
+    init_telemetry,
+    instrument_fastapi,
+    origin_links,
+)
 
-logger = logging.getLogger("bitrix-local-app")
+logger = structlog.get_logger("bitrix-local-app")
+EXPECTED_BITRIX_DB_REVISION = "0002_inbox_trace_context"
 
 BITRIX_SENDER_PREFIX = re.compile(
     r"^\[b\][^\r\n\[]+:\[/b\]\s*(?:\[br\]\s*)?",
@@ -57,6 +69,7 @@ CONNECTOR_ICON_DATA_URI = "data:image/svg+xml," + quote(
 class Settings(BaseSettings):
     model_config = SettingsConfigDict(extra="ignore")
     app_env: str = "production-like"
+    log_level: str = "INFO"
     bitrix_database_url: str
     bitrix_client_id: str
     bitrix_client_secret: str
@@ -78,6 +91,28 @@ class Settings(BaseSettings):
     bitrix_worker_poll_sec: float = Field(default=1, ge=0.05, le=30)
 
 
+def configure_logging(level: str, telemetry: TelemetryRuntime | None = None) -> None:
+    logging.basicConfig(level=level, format="%(message)s")
+    if telemetry and telemetry.logging_handler not in logging.getLogger().handlers:
+        telemetry.logging_handler.addFilter(
+            lambda record: not record.name.startswith("opentelemetry")
+        )
+        logging.getLogger().addHandler(telemetry.logging_handler)
+    structlog.configure(
+        processors=[
+            structlog.contextvars.merge_contextvars,
+            add_trace_context,
+            redact_event,
+            structlog.processors.TimeStamper(fmt="iso", utc=True, key="timestamp"),
+            structlog.stdlib.add_log_level,
+            structlog.processors.JSONRenderer(),
+        ],
+        logger_factory=structlog.stdlib.LoggerFactory(),
+        wrapper_class=structlog.stdlib.BoundLogger,
+        cache_logger_on_first_use=True,
+    )
+
+
 class TokenCipher:
     def __init__(self, encoded_key: str, version: str) -> None:
         try:
@@ -309,8 +344,8 @@ def normalize_event(payload: dict[str, Any]) -> dict[str, Any] | None:
                 normalized_file["_bitrix_file_id"] = str(file_id)
             if not normalized_file["download_url"] and not file_id:
                 logger.warning(
-                    "inbound file has no supported download reference; keys=%s",
+                    "inbound_file.unsupported_reference",
-                    sorted(str(key) for key in item),
+                    keys=sorted(str(key) for key in item),
                 )
             files.append(normalized_file)
         if not text_value and not files:
@@ -498,6 +533,8 @@ def create_app(settings: Settings | None = None) -> FastAPI:
 
     @asynccontextmanager
     async def lifespan(app: FastAPI):
+        telemetry = init_telemetry()
+        configure_logging(cfg.log_level, telemetry)
         engine = create_postgres_engine(
             cfg.bitrix_database_url,
             pool_size=5,
@@ -527,6 +564,8 @@ def create_app(settings: Settings | None = None) -> FastAPI:
                 await task
         await app.state.http.aclose()
         await engine.dispose()
+        if telemetry:
+            telemetry.shutdown()
 
     app = FastAPI(
         title="HAN Bitrix24 Local App",
@@ -540,6 +579,10 @@ def create_app(settings: Settings | None = None) -> FastAPI:
     @app.middleware("http")
     async def request_context(request: Request, call_next):
         request.state.request_id = request.headers.get("X-Request-ID") or str(uuid.uuid4())
+        with structlog.contextvars.bound_contextvars(
+            request_id=request.state.request_id,
+            **{"service.name": "bitrix-local-app"},
+        ):
             response = await call_next(request)
             response.headers["X-Request-ID"] = request.state.request_id
             return response
@@ -579,9 +622,22 @@ def create_app(settings: Settings | None = None) -> FastAPI:
             async with request.app.state.sessions() as session:
                 portal = await active_portal(session)
                 await session.scalar(select(func.now()))
+                revision = await session.scalar(
+                    text("SELECT version_num FROM bitrix_local.alembic_version")
+                )
             worker_ok = all(not task.done() for task in request.app.state.workers)
-            if portal and portal.install_status == "installed" and worker_ok:
+            if (
+                portal
+                and portal.install_status == "installed"
+                and worker_ok
+                and revision == EXPECTED_BITRIX_DB_REVISION
+            ):
                 return {"status": "ready", "portal": "installed", "workers": "running"}
+            if revision != EXPECTED_BITRIX_DB_REVISION:
+                return JSONResponse(
+                    {"status": "not_ready", "reason": "migration_required"},
+                    status_code=503,
+                )
             return JSONResponse(
                 {"status": "not_ready", "reason": "portal_not_installed"}, status_code=503
             )
@@ -655,7 +711,12 @@ def create_app(settings: Settings | None = None) -> FastAPI:
             normalized = normalize_event(payload)
             if normalized is None:
                 return {"status": "ignored"}
-            created = await save_inbox(request.app, normalized)
+            created = await save_inbox(
+                request.app,
+                normalized,
+                request.state.request_id,
+                current_traceparent(),
+            )
             return JSONResponse(
                 {"status": "accepted" if created else "duplicate"},
                 status_code=202 if created else 200,
@@ -731,6 +792,8 @@ def create_app(settings: Settings | None = None) -> FastAPI:
                     external_chat_id=dto.external_chat_id,
                     request_fingerprint=fp,
                     payload_json=body,
+                    request_id=request.state.request_id,
+                    traceparent=current_traceparent(),
                     status="sending",
                     lease_until=now() + timedelta(seconds=cfg.bitrix_http_timeout_sec + 5),
                 )
@@ -768,12 +831,10 @@ def create_app(settings: Settings | None = None) -> FastAPI:
             )
         except Exception as exc:
             logger.error(
-                "outbound delivery failed",
+                "outbound.delivery_failed",
-                extra={
+                request_id=request.state.request_id,
-                    "request_id": request.state.request_id,
+                message_id=str(dto.message_id),
-                    "message_id": str(dto.message_id),
+                error_type=type(exc).__name__,
-                    "error_type": type(exc).__name__,
-                },
             )
             async with request.app.state.sessions() as session:
                 current = await session.get(OutboundMessage, row.id, with_for_update=True)
@@ -845,6 +906,7 @@ def create_app(settings: Settings | None = None) -> FastAPI:
         result = await reconcile_setup(request.app)
         return {"status": "completed" if all(result.values()) else "partial", "steps": result}
 
+    instrument_fastapi(app)
     return app
 
 
@@ -960,7 +1022,12 @@ async def uninstall_payload(app: FastAPI, payload: dict[str, Any]) -> None:
             await session.commit()
 
 
-async def save_inbox(app: FastAPI, normalized: dict[str, Any]) -> bool:
+async def save_inbox(
+    app: FastAPI,
+    normalized: dict[str, Any],
+    request_id: str,
+    traceparent: str | None,
+) -> bool:
     fingerprint = hashlib.sha256(
         json.dumps(normalized, sort_keys=True, separators=(",", ":")).encode()
     ).hexdigest()
@@ -971,6 +1038,8 @@ async def save_inbox(app: FastAPI, normalized: dict[str, Any]) -> bool:
         bitrix_message_id=normalized["bitrix_message_id"],
         payload_fingerprint=fingerprint,
         normalized_json=normalized,
+        request_id=request_id,
+        traceparent=traceparent,
     )
     async with app.state.sessions() as session:
         session.add(row)
@@ -1168,8 +1237,9 @@ async def worker_loop(app: FastAPI, kind: str) -> None:
                 await process_setup(app)
         except Exception as exc:
             logger.error(
-                "worker iteration failed",
+                "worker.iteration_failed",
-                extra={"worker_kind": kind, "error_type": type(exc).__name__},
+                worker_kind=kind,
+                error_type=type(exc).__name__,
             )
         try:
             await asyncio.wait_for(app.state.stop.wait(), app.state.settings.bitrix_worker_poll_sec)
@@ -1182,6 +1252,18 @@ async def process_inbox(app: FastAPI) -> None:
         row = await claim_one(session, InboxEvent, ["received", "retry"])
     if not row:
         return
+    tracer = trace.get_tracer("han.bitrix.inbox")
+    with (
+        tracer.start_as_current_span(
+            "bitrix.inbox.forward",
+            links=origin_links(row.traceparent),
+        ),
+        structlog.contextvars.bound_contextvars(
+            request_id=row.request_id,
+            inbox_id=str(row.id),
+            **{"service.name": "bitrix-local-app"},
+        ),
+    ):
         try:
             payload = json.loads(json.dumps(row.normalized_json))
             await resolve_inbound_file_urls(app, payload)
@@ -1190,7 +1272,7 @@ async def process_inbox(app: FastAPI) -> None:
                 json=payload,
                 headers={
                     "Authorization": f"Bearer {app.state.settings.bitrix_api_forward_token}",
-                "X-Request-ID": str(uuid.uuid4()),
+                    "X-Request-ID": row.request_id or str(uuid.uuid4()),
                 },
                 follow_redirects=False,
             )
@@ -1205,6 +1287,8 @@ async def process_inbox(app: FastAPI) -> None:
                 session.add(
                     DeliveryAckOutbox(
                         inbox_event_id=current.id,
+                        request_id=current.request_id,
+                        traceparent=current.traceparent,
                         payload_json={
                             "external_chat_id": str(current.external_chat_id),
                             "bitrix_message_id": current.bitrix_message_id,
@@ -1214,11 +1298,9 @@ async def process_inbox(app: FastAPI) -> None:
                 await session.commit()
         except Exception as exc:
             logger.error(
-            "inbound forward failed",
+                "inbound.forward_failed",
-            extra={
+                inbox_id=str(row.id),
-                "inbox_id": str(row.id),
+                error_type=type(exc).__name__,
-                "error_type": type(exc).__name__,
-            },
             )
             await mark_retry(app, InboxEvent, row.id, "api_forward_failed")
 
@@ -1252,6 +1334,18 @@ async def process_ack(app: FastAPI) -> None:
         portal = await active_portal(session)
     if not row or not portal:
         return
+    tracer = trace.get_tracer("han.bitrix.ack")
+    with (
+        tracer.start_as_current_span(
+            "bitrix.delivery_ack.process",
+            links=origin_links(row.traceparent),
+        ),
+        structlog.contextvars.bound_contextvars(
+            request_id=row.request_id,
+            ack_outbox_id=str(row.id),
+            **{"service.name": "bitrix-local-app"},
+        ),
+    ):
         try:
             await app.state.bitrix.call(
                 portal,
@@ -1265,7 +1359,9 @@ async def process_ack(app: FastAPI) -> None:
             )
             async with app.state.sessions() as session:
                 current = await session.get(DeliveryAckOutbox, row.id, with_for_update=True)
-            event = await session.get(InboxEvent, current.inbox_event_id, with_for_update=True)
+                event = await session.get(
+                    InboxEvent, current.inbox_event_id, with_for_update=True
+                )
                 current.status = "completed"
                 current.lease_until = None
                 event.status = "completed"
@@ -1285,6 +1381,18 @@ async def process_outbound(app: FastAPI) -> None:
         )
     if not row:
         return
+    tracer = trace.get_tracer("han.bitrix.outbound")
+    with (
+        tracer.start_as_current_span(
+            "bitrix.outbound.process",
+            links=origin_links(row.traceparent),
+        ),
+        structlog.contextvars.bound_contextvars(
+            request_id=row.request_id,
+            outbound_id=str(row.id),
+            **{"service.name": "bitrix-local-app"},
+        ),
+    ):
         try:
             await asyncio.wait_for(
                 deliver_outbound(app, row.id),
@@ -1292,11 +1400,9 @@ async def process_outbound(app: FastAPI) -> None:
             )
         except Exception as exc:
             logger.error(
-            "outbound retry failed",
+                "outbound.retry_failed",
-            extra={
+                outbound_id=str(row.id),
-                "outbound_id": str(row.id),
+                error_type=type(exc).__name__,
-                "error_type": type(exc).__name__,
-            },
             )
             await mark_retry(app, OutboundMessage, row.id, "bitrix_delivery_failed")
 

VM1_app/codebase/backend/bitrix-local-app/app/models.py

@@ -133,6 +133,8 @@ class InboxEvent(Common, Base):
     bitrix_message_id: Mapped[str | None] = mapped_column(String(255))
     payload_fingerprint: Mapped[str] = mapped_column(String(64))
     normalized_json: Mapped[dict] = mapped_column(JSON)
+    request_id: Mapped[str | None] = mapped_column(String(64))
+    traceparent: Mapped[str | None] = mapped_column(String(55))
     status: Mapped[str] = mapped_column(String(32), default="received")
     attempt_count: Mapped[int] = mapped_column(Integer, default=0)
     next_attempt_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=now)
@@ -149,6 +151,8 @@ class OutboundMessage(Common, Base):
     external_chat_id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True))
     request_fingerprint: Mapped[str] = mapped_column(String(64))
     payload_json: Mapped[dict] = mapped_column(JSON)
+    request_id: Mapped[str | None] = mapped_column(String(64))
+    traceparent: Mapped[str | None] = mapped_column(String(55))
     status: Mapped[str] = mapped_column(String(32), default="received")
     bitrix_message_id: Mapped[str | None] = mapped_column(String(255))
     response_json: Mapped[dict | None] = mapped_column(JSON)
@@ -165,6 +169,8 @@ class DeliveryAckOutbox(Common, Base):
         UUID(as_uuid=True), ForeignKey(f"{SCHEMA}.inbox_events.id"), unique=True
     )
     payload_json: Mapped[dict] = mapped_column(JSON)
+    request_id: Mapped[str | None] = mapped_column(String(64))
+    traceparent: Mapped[str | None] = mapped_column(String(55))
     status: Mapped[str] = mapped_column(String(32), default="pending")
     attempt_count: Mapped[int] = mapped_column(Integer, default=0)
     next_attempt_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=now)

VM1_app/codebase/backend/bitrix-local-app/app/telemetry.py

@@ -0,0 +1,119 @@
+from __future__ import annotations
+
+import logging
+import os
+from collections.abc import MutableMapping
+from dataclasses import dataclass
+from typing import Any
+
+from fastapi import FastAPI
+from opentelemetry import metrics, trace
+from opentelemetry._logs import set_logger_provider
+from opentelemetry.exporter.otlp.proto.grpc._log_exporter import OTLPLogExporter
+from opentelemetry.exporter.otlp.proto.grpc.metric_exporter import OTLPMetricExporter
+from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter
+from opentelemetry.instrumentation.fastapi import FastAPIInstrumentor
+from opentelemetry.instrumentation.httpx import HTTPXClientInstrumentor
+from opentelemetry.instrumentation.sqlalchemy import SQLAlchemyInstrumentor
+from opentelemetry.propagate import extract, inject, set_global_textmap
+from opentelemetry.sdk._logs import LoggerProvider, LoggingHandler
+from opentelemetry.sdk._logs.export import BatchLogRecordProcessor
+from opentelemetry.sdk.metrics import MeterProvider
+from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader
+from opentelemetry.sdk.resources import Resource
+from opentelemetry.sdk.trace import TracerProvider
+from opentelemetry.sdk.trace.export import BatchSpanProcessor
+from opentelemetry.sdk.trace.sampling import ALWAYS_ON
+from opentelemetry.trace.propagation.tracecontext import TraceContextTextMapPropagator
+
+
+@dataclass(slots=True)
+class TelemetryRuntime:
+    tracer_provider: TracerProvider
+    meter_provider: MeterProvider
+    logger_provider: LoggerProvider
+    logging_handler: LoggingHandler
+
+    def shutdown(self) -> None:
+        self.logger_provider.shutdown()
+        self.meter_provider.shutdown()
+        self.tracer_provider.shutdown()
+
+
+_runtime: TelemetryRuntime | None = None
+
+
+def init_telemetry() -> TelemetryRuntime | None:
+    global _runtime
+    if _runtime is not None:
+        return _runtime
+    endpoint = os.getenv("OTEL_EXPORTER_OTLP_ENDPOINT", "").strip()
+    if not endpoint:
+        return None
+    resource = Resource.create(
+        {
+            "service.name": os.getenv("OTEL_SERVICE_NAME", "bitrix-local-app"),
+            "service.namespace": "han-chat",
+            "service.version": os.getenv("RELEASE_VERSION", "unknown"),
+            "deployment.environment": os.getenv("APP_ENV", "production-like"),
+        }
+    )
+    insecure = endpoint.startswith("http://")
+    set_global_textmap(TraceContextTextMapPropagator())
+    tracer_provider = TracerProvider(resource=resource, sampler=ALWAYS_ON)
+    tracer_provider.add_span_processor(
+        BatchSpanProcessor(OTLPSpanExporter(endpoint=endpoint, insecure=insecure, timeout=3))
+    )
+    trace.set_tracer_provider(tracer_provider)
+    metric_reader = PeriodicExportingMetricReader(
+        OTLPMetricExporter(endpoint=endpoint, insecure=insecure, timeout=3),
+        export_interval_millis=30000,
+        export_timeout_millis=3000,
+    )
+    meter_provider = MeterProvider(resource=resource, metric_readers=[metric_reader])
+    metrics.set_meter_provider(meter_provider)
+    logger_provider = LoggerProvider(resource=resource)
+    logger_provider.add_log_record_processor(
+        BatchLogRecordProcessor(
+            OTLPLogExporter(endpoint=endpoint, insecure=insecure, timeout=3),
+            max_queue_size=2048,
+            max_export_batch_size=512,
+            export_timeout_millis=3000,
+        )
+    )
+    set_logger_provider(logger_provider)
+    logging_handler = LoggingHandler(level=logging.NOTSET, logger_provider=logger_provider)
+    HTTPXClientInstrumentor().instrument()
+    SQLAlchemyInstrumentor().instrument(enable_commenter=False)
+    _runtime = TelemetryRuntime(
+        tracer_provider, meter_provider, logger_provider, logging_handler
+    )
+    return _runtime
+
+
+def instrument_fastapi(app: FastAPI) -> None:
+    FastAPIInstrumentor.instrument_app(app, excluded_urls="/health/live,/health/ready")
+
+
+def add_trace_context(
+    _logger: Any, _method_name: str, event_dict: MutableMapping[str, Any]
+) -> MutableMapping[str, Any]:
+    context = trace.get_current_span().get_span_context()
+    if context.is_valid:
+        event_dict["trace_id"] = format(context.trace_id, "032x")
+        event_dict["span_id"] = format(context.span_id, "016x")
+    return event_dict
+
+
+def current_traceparent() -> str | None:
+    carrier: dict[str, str] = {}
+    inject(carrier)
+    return carrier.get("traceparent")
+
+
+def origin_links(traceparent: str | None) -> list[trace.Link]:
+    if not traceparent:
+        return []
+    context = extract({"traceparent": traceparent})
+    span_context = trace.get_current_span(context).get_span_context()
+    return [trace.Link(span_context)] if span_context.is_valid else []

VM1_app/codebase/backend/bitrix-local-app/pyproject.toml

@@ -8,9 +8,16 @@ dependencies = [
   "cryptography>=45,<46",
   "fastapi>=0.116,<1",
   "httpx>=0.28,<1",
+  "opentelemetry-api>=1.44,<2",
+  "opentelemetry-exporter-otlp-proto-grpc>=1.44,<2",
+  "opentelemetry-instrumentation-fastapi>=0.65b0,<1",
+  "opentelemetry-instrumentation-httpx>=0.65b0,<1",
+  "opentelemetry-instrumentation-sqlalchemy>=0.65b0,<1",
+  "opentelemetry-sdk>=1.44,<2",
   "pydantic-settings>=2.10,<3",
   "python-multipart>=0.0.20,<1",
   "sqlalchemy[asyncio]>=2.0.41,<3",
+  "structlog>=25,<26",
   "uvicorn[standard]>=0.35,<1",
 ]
 

VM1_app/codebase/backend/bitrix-local-app/tests/test_core.py

@@ -19,6 +19,7 @@ os.environ.setdefault(
 import httpx
 import pytest
 
+from app.logging_security import REDACTED, sanitize_value
 from app.main import (
     BitrixClient,
     TokenCipher,
@@ -30,6 +31,17 @@ from app.main import (
     safely_retryable,
     validate_portal,
 )
+from app.telemetry import add_trace_context, origin_links
+
+
+def test_bitrix_logs_redact_secrets_and_do_not_invent_trace_ids():
+    value = sanitize_value(
+        {"authorization": "Bearer secret", "url": "https://example.test/?token=secret"}
+    )
+    assert value["authorization"] == REDACTED
+    assert "secret" not in value["url"]
+    assert add_trace_context(None, "info", {"event": "safe"}) == {"event": "safe"}
+    assert origin_links("invalid") == []
 
 
 def test_token_cipher_binds_aad():

VM1_app/codebase/backend/deployment/RUNBOOK.production.ru.md

@@ -339,6 +339,21 @@ downgrade запрещены.
 
 ## 9. Упорядоченный первый запуск
 
+До preflight установите host Collector из pinned release artifact. Версию и
+SHA-256 возьмите из утверждённого release record, не из ответа GitHub API:
+
+```sh
+export OTEL_HOST_COLLECTOR_VERSION='0.117.0'
+export OTEL_HOST_COLLECTOR_SHA256='90710c909a30fc3b89dd0e389c13f9e57ebbb87d6a0dc51fdde0bf03499a5f25'
+deployment/scripts/setup-vm.sh
+/usr/local/bin/otelcol-contrib validate \
+  --config=deployment/observability/otel-host-collector.yaml
+```
+
+`setup-vm.sh` устанавливает и enable-ит unit, но не запускает host Collector.
+Он работает без Docker socket; offsets и exporter queue находятся в
+`/var/lib/han-otel/host-collector`.
+
 Первый запуск выполняет root через фиксированный launcher:
 
 ```sh
@@ -363,6 +378,8 @@ docker kill --signal HUP "$NGINX_ID" >/dev/null
 unset NGINX_ID
 systemctl enable han-secrets@production.service han-stack@production.service
 systemctl start han-stack@production.service
+systemctl start han-host-otel-collector@production.service
+systemctl --no-pager status han-host-otel-collector@production.service
 ```
 
 `han-stack@production` становится единственным routine lifecycle interface.
@@ -435,6 +452,13 @@ staging, выполнять config test и HUP.
 5xx/auth/Safety/PG/Redis/OOM/disk/OTEL queue/TLS. Отправьте только fake canary
 token/PII markers и докажите их отсутствие в logs/traces.
 
+В SigNoz один canary request должен дать ровно по одному application log от
+nginx/API/Bitrix, а `trace_id`/`span_id` application logs должны открывать
+соответствующий span. Platform logs ищутся по `host.name`, `service.name` и
+временному окну. Проверьте отсутствие `unknown-container`, дублей Python logs,
+логов обоих Collector и экспоненциального роста ingest. После restart host
+Collector старые записи не должны replay-иться: offsets сохраняются.
+
 KESL 12.4 устанавливается и принимается только по отдельному операторскому
 runbook `deployment/kesl/RUNBOOK.KESL.ru.md`. Не совмещайте установку,
 полную/контейнерную антивирусную проверку или изменение File Threat Protection
@@ -445,7 +469,8 @@ runbook `deployment/kesl/RUNBOOK.KESL.ru.md`. Не совмещайте уста
 
 ```sh
 systemctl is-enabled docker.service han-chat-docker-firewall.service \
-  han-secrets@production.service han-stack@production.service certbot.timer
+  han-secrets@production.service han-stack@production.service \
+  han-host-otel-collector@production.service certbot.timer
 systemctl reboot
 ```
 
@@ -466,10 +491,13 @@ systemctl daemon-reload
 systemctl restart han-secrets@production.service
 /opt/han-chat/current/backend/deployment/preflight.sh
 systemctl restart han-stack@production.service
+systemctl restart han-host-otel-collector@production.service
 ```
 
 Повторите smoke и зафиксируйте digests. Не удаляйте current/previous release,
-active images, evidence или volumes. После incompatible migration используйте
+active images, evidence, volumes или `/var/lib/han-otel`: offsets/queue должны
+пережить rollback. Если previous release не содержит совместимого host config,
+остановите host unit до возврата совместимой версии. После incompatible migration используйте
 forward fix либо согласованный PG PITR + S3/Bitrix reconciliation в maintenance
 window; Redis восстанавливается пустым и прогревается из PG.
 

VM1_app/codebase/backend/deployment/han-host-otel-collector@.service

@@ -0,0 +1,45 @@
+[Unit]
+Description=HAN Chat VM1 host telemetry collector (%i)
+Requires=han-secrets@%i.service
+After=docker.service han-secrets@%i.service network-online.target
+Wants=docker.service network-online.target
+ConditionPathIsExecutable=/usr/local/bin/otelcol-contrib
+ConditionPathIsDirectory=/opt/han-chat/current/backend
+
+[Service]
+Type=simple
+User=root
+Group=root
+EnvironmentFile=/etc/han/vm1.env
+ExecStartPre=/usr/local/bin/otelcol-contrib validate --config=/opt/han-chat/current/backend/deployment/observability/otel-host-collector.yaml
+ExecStart=/usr/local/bin/otelcol-contrib --config=/opt/han-chat/current/backend/deployment/observability/otel-host-collector.yaml
+Restart=on-failure
+RestartSec=10s
+TimeoutStopSec=30
+UMask=0077
+NoNewPrivileges=yes
+PrivateTmp=yes
+ProtectSystem=strict
+ProtectHome=yes
+ProtectKernelTunables=yes
+ProtectKernelModules=yes
+ProtectKernelLogs=yes
+ProtectControlGroups=yes
+ProtectClock=yes
+RestrictRealtime=yes
+RestrictSUIDSGID=yes
+LockPersonality=yes
+CapabilityBoundingSet=
+AmbientCapabilities=
+RestrictAddressFamilies=AF_UNIX AF_INET AF_INET6
+ReadOnlyPaths=/opt/han-chat/current/backend/deployment/observability
+ReadOnlyPaths=/var/lib/docker/containers
+ReadOnlyPaths=/var/log/journal
+ReadOnlyPaths=/run/log/journal
+ReadOnlyPaths=/run/han-chat/secrets
+ReadWritePaths=/var/lib/han-otel/host-collector
+LimitCORE=0
+LimitNOFILE=65536
+
+[Install]
+WantedBy=multi-user.target

VM1_app/codebase/backend/deployment/kesl/EVIDENCE.AVZ.ru.md

@@ -4,52 +4,61 @@
 должна содержать activation code, ключи, токены, DSN, environment, содержимое
 secret-файлов, персональные данные или тестовый файл EICAR.
 
+Ниже зафиксирован **тестовый пилот** на `devhanapp` (4 ГБ RAM). Это не
+автоматическая приёмка боевой среды.
+
 ## 1. Идентификация изменения
 
-- Change ID:
+- Change ID: тестовый пилот KESL 12.4 на ВМ1, 2026-09-07
-- Дата и окно:
+- Дата и окно: 2026-09-07 11:03–11:43 MSK
-- Оператор:
+- Оператор: root на тестовой ВМ (сессия оператора)
-- Security approver:
+- Security approver: не подписано в этом файле
-- Service owner:
+- Service owner: не подписано в этом файле
-- Hostname ВМ1:
+- Hostname ВМ1: `devhanapp`
-- Ubuntu version:
+- Ubuntu version: Ubuntu 24.04.4 LTS
-- Kernel version:
+- Kernel version: `6.8.0-138-generic` x86_64
-- KESL package/version:
+- KESL package/version: `kesl` `12.4.0-1225` amd64
-- SHA-256 DEB:
+- SHA-256 DEB: `e0befb5dbf628344ecf022259841f9e3688e86b07ce094f45ed871711b98fc7f`
-- Источник пакета:
+- Источник пакета: ISO `049-16-d-01.iso` (volume id `KESL12SP4`)
-- HAN release SHA:
+- SHA-256 ISO: `8599a7ed8d7d661a70811d668e5f98e372b674c87ef4cd24abbd917625a2d5e5`
-- Container image digests зафиксированы: да / нет
+- ГОСТ Р 34.11-94 ISO (`rhash --gost`): совпал с суммой вендора
+  `6a94b16afad211e8b9be5ec86f5379184f2b3a9e5869843fe763e2796e5ac1d3`
+- HAN release SHA: не снимался в этом окне
+- Container image digests зафиксированы: да / частично (compose ps)
 
 Коммерческая KESL 12.4 не должна быть обозначена как сертифицированная ФСТЭК
 сборка. Решение о допустимости коммерческой версии и ссылка на модель угроз:
 
-- Решение:
+- Решение: коммерческая 12.4 выбрана из‑за Ubuntu 24.04; сертифицированную
-- Документ/раздел:
+  сборку не заявлять
-- Утвердил:
+- Документ/раздел: модель угроз / акт СЗПД — заполняет Security
+- Утвердил: не подписано в этом файле
 
 ## 2. Входной baseline
 
-- Все steady-state контейнеры healthy/running:
+- Все steady-state контейнеры healthy/running: да (13 running / 15 total,
-- Restart count:
+  2 oneshot)
-- Public smoke:
+- Restart count: без новых restart в окне пилота
-- Negative port probes:
+- Public smoke: в этом окне не повторялся
-- CPU:
+- Negative port probes: в этом окне не повторялись
-- Available RAM:
+- CPU: контейнеры < 4% кроме кратких всплесков
-- Swap activity:
+- Available RAM: до KESL ~1.9 ГБ; после пилота ~2.3 ГБ (часть ушла в swap)
-- Disk free:
+- Swap activity: до установки ~524 КиБ; после ~675 МБ
-- IO wait:
+- Disk free: после growpart 19 ГБ из 30 ГБ
-- API p95:
+- IO wait: не снимался отдельно
-- Redis latency / blocked clients:
+- API p95: не снимался
-- OTEL queue:
+- Redis latency / blocked clients: не снимались
-- Открытые до установки проблемы:
+- OTEL queue: не снималась
+- Открытые до установки проблемы: публичный SSH 22; RAM 4 ГБ ниже
+  production-gate
 
 Stop conditions и численные пороги утверждены:
 
-- p95/Redis:
+- p95/Redis: для теста не применялись
-- available RAM/swap:
+- available RAM/swap: stop при available < 512 МБ; не достигнуто
-- IO wait:
+- IO wait: не применялся
-- disk:
+- disk: свободно > 10 ГБ
-- health/restarts:
+- health/restarts: новых unhealthy не было
 
 ## 3. АВЗ.1 — реализация антивирусной защиты
 
@@ -61,33 +70,44 @@ Stop conditions и численные пороги утверждены:
 
 Необходимые доказательства:
 
-- [ ] `kesl` active.
+- [x] `kesl` active.
-- [ ] Лицензия действительна.
+- [x] Лицензия действительна (`The key is valid`, subscription active).
-- [ ] File Threat Protection (ID 1) имеет состояние `Started`.
+- [x] File Threat Protection (ID 1) имеет состояние `Started`.
-- [ ] InterceptorProtectionMode = `Block`.
+- [x] Перехватчик: fanotify, штатный блокирующий режим (параметр
-- [ ] ActionOnThreat = `DisinfectDeleteIfNotPossible` либо иное утверждённое
+      `InterceptorProtectionMode` на этой ОС не поддерживается).
+- [x] ActionOnThreat = `DisinfectDeleteIfNotPossible` либо иное утверждённое
       блокирующее/лечащее действие.
-- [ ] ScanArchived = `No` для real-time защиты.
+- [x] ScanArchived = `No` для real-time защиты.
-- [ ] Исключения ограничены тремя утверждёнными hot-data mountpoint.
+- [x] Исключения ограничены тремя утверждёнными hot-data mountpoint.
-- [ ] Контролируемый EICAR заблокирован/обезврежен/помещён в карантин.
+- [x] Контролируемый EICAR заблокирован/обезврежен/помещён в карантин.
-- [ ] Событие EICAR зарегистрировано в журнале KESL.
+- [x] Событие EICAR зарегистрировано в журнале KESL.
-- [ ] После теста EICAR отсутствует вне карантина и тестовый каталог удалён.
+- [x] После теста EICAR отсутствует вне карантина и тестовый каталог удалён.
 - [ ] Public smoke и health после включения Block успешны.
-- [ ] UFW и `HAN-CHAT-DOCKER` не изменены.
+      (health контейнеров подтверждён; внешний smoke не запускался)
+- [x] UFW и `HAN-CHAT-DOCKER` не изменены.
 - [ ] За 24 часа нет новых restart/OOM/5xx и неприемлемой деградации.
 
 Артефакты без секретов:
 
-- `systemctl is-active kesl`:
+- `systemctl is-active kesl`: `active`
-- `kesl-control --app-info`:
+- `kesl-control --app-info`: 12.4.0.1225; key valid; databases loaded Yes;
-- `kesl-control --get-task-state 1`:
+  File Threat Protection Available and running
-- reviewed excerpt `kesl-control --get-settings 1`:
+- `kesl-control --get-task-state 1`: `Started`
-- EICAR event ID/time/action:
+- reviewed excerpt `kesl-control --get-settings 1`: ScanArchived=No;
-- smoke result/time:
+  ActionOnThreat=DisinfectDeleteIfNotPossible; ScanByAccessType=SmartCheck;
-- firewall comparison:
+  три ExcludedFromScanScope
-- 24h resource comparison:
+- EICAR event ID/time/action: EventId 3240 `ThreatDetected` /
+  `EICAR-Test-File` 2026-09-07 11:43:45; 3241 Backup; 3242 NotDisinfected
+  NonCurable; 3243 ObjectDeleted. Сработало **Scan_File (ODS)**, не OAS
+  on-access: файл успели записать на диск до scan-file.
+- smoke result/time: не выполнялся
+- firewall comparison: `HAN-CHAT-DOCKER` без новых правил KESL
+- 24h resource comparison: не выдержан
 
-Вывод по АВЗ.1: реализована / не реализована.
+Вывод по АВЗ.1: **реализована на тестовой ВМ** (обнаружение + backup +
+удаление). Ограничение: on-access не перехватил создание EICAR; реакция
+доказана on-demand. Для боя повторить on-access (запись + чтение файла)
+на ресурсах ≥16 ГБ.
 
 ## 4. АВЗ.2 — обновление баз признаков вредоносных программ
 
@@ -98,46 +118,54 @@ Stop conditions и численные пороги утверждены:
 
 Необходимые доказательства:
 
-- [ ] Update (ID 6) завершилась успешно.
+- [x] Update (ID 6) завершилась успешно.
-- [ ] Базы загружены.
+- [x] Базы загружены.
-- [ ] Дата выпуска баз актуальна на момент проверки.
+- [x] Дата выпуска баз актуальна на момент проверки.
-- [ ] Расписание Update = `Hourly`.
+- [x] Расписание Update = `Hourly` (`StartTime=2026/Sep/07 11:32:38;1`).
 - [ ] Утверждён alert/регламент на ошибку и устаревание баз.
 - [ ] Назначен ответственный за ежедневный контроль.
 - [ ] Проверено успешное автоматическое обновление после ручного запуска.
+      (ручной Update успешен; первый hourly цикл ещё не наблюдался)
 
 Артефакты без секретов:
 
-- `kesl-control --app-info`:
+- `kesl-control --app-info`: databases loaded Yes; last release
-- `kesl-control --get-task-state 6`:
+  2026-09-07 11:25:00
-- `kesl-control --get-schedule 6`:
+- `kesl-control --get-task-state 6`: Stopped после успешного ручного запуска
-- время последнего успешного автоматического Update:
+- `kesl-control --get-schedule 6`: RuleType=Hourly; interval 1 hour
-- ссылка на alert/регламент:
+- время последнего успешного автоматического Update: не наблюдалось
-- ответственный:
+- ссылка на alert/регламент: не создан
+- ответственный: не назначен
 
-Вывод по АВЗ.2: реализована / не реализована.
+Вывод по АВЗ.2: **реализована на тестовой ВМ** (ручное обновление +
+почасовое расписание). Для боя нужны наблюдаемый hourly цикл и alert на сбой.
 
 ## 5. Связанные меры
 
 ### РСБ.1–3, РСБ.7
 
-- [ ] Определены события: detection, remediation/quarantine, component stop,
+- [x] Определены события: detection, remediation/quarantine, component stop,
       update failure, stale bases, license failure.
-- [ ] Определён состав полей: time, host, component/task, threat, object,
+      (фиксируются в журнале KESL; пример ThreatDetected/ObjectDeleted)
+- [x] Определён состав полей: time, host, component/task, threat, object,
       action, result, severity.
 - [ ] Определены срок и место хранения.
-- [ ] Доступ к журналу ограничен; изменение/удаление контролируется.
+      (по умолчанию `/var/opt/kaspersky/kesl/private/storage/events.db`)
+- [x] Доступ к журналу ограничен; изменение/удаление контролируется.
+      (root-only `kesl-control -E`)
 - [ ] Экспорт в syslog/SIEM включён либо документирован локальный контроль.
 
-Ссылка на регламент и настройки:
+Ссылка на регламент и настройки: локальный журнал KESL; SIEM не подключался.
 
 ### АНЗ.2
 
-- [ ] Контролируется версия и жизненный цикл самого KESL, а не только баз.
+- [x] Контролируется версия и жизненный цикл самого KESL, а не только баз.
-- [ ] Upgrade KESL проходит совместимость, pilot, smoke и rollback review.
+      (12.4.0.1225 зафиксирован; после Update был restart модуля)
-- [ ] Обновление kernel/Docker вызывает повторную проверку совместимости.
+- [x] Upgrade KESL проходит совместимость, pilot, smoke и rollback review.
+      (процедура в `RUNBOOK.KESL.ru.md`)
+- [x] Обновление kernel/Docker вызывает повторную проверку совместимости.
 
-Ссылка на регламент:
+Ссылка на регламент: `deployment/kesl/RUNBOOK.KESL.ru.md`
 
 ## 6. Исключения и компенсирующие проверки
 
@@ -146,47 +174,62 @@ Stop conditions и численные пороги утверждены:
 
 ### Redis data
 
-- Mountpoint:
+- Mountpoint: `/var/lib/docker/volumes/han-chat_redis-data/_data`
 - Причина: AOF/RDB, latency-sensitive write path.
-- Компенсация:
+- Компенсация: host File Threat Protection вне volume; on-demand scan-file
-- Владелец:
+  release; container monitoring недоступен по лицензии
-- Review date:
+- Владелец: Operations ВМ1
+- Review date: при переносе на боевые ресурсы / не позднее 1 месяца
 
 ### OTEL queue
 
-- Mountpoint:
+- Mountpoint: `/var/lib/docker/volumes/han-chat_otel-queue/_data`
 - Причина: persistent high-churn telemetry queue.
-- Компенсация:
+- Компенсация: как выше
-- Владелец:
+- Владелец: Operations ВМ1
-- Review date:
+- Review date: при переносе на боевые ресурсы / не позднее 1 месяца
 
 ### nginx cache
 
-- Mountpoint:
+- Mountpoint: `/var/lib/docker/volumes/han-chat_nginx-cache/_data`
 - Причина: regenerable high-churn cache.
-- Компенсация:
+- Компенсация: как выше
-- Владелец:
+- Владелец: Operations ВМ1
-- Review date:
+- Review date: при переносе на боевые ресурсы / не позднее 1 месяца
 
 Иных исключений нет / перечислить отдельно с утверждением Security:
+иных исключений нет. `/var/lib/docker` целиком не исключался.
+
+Лицензионное ограничение: Container Monitoring / Container Scan недоступны.
 
 ## 7. Проверка отката
 
-- [ ] Команда переключения `Block` → `Notify` проверена документально.
+- [x] Команда `kesl-control --stop-task 1` проверена документально.
-- [ ] Процедура остановки KESL доступна break-glass admin.
+- [x] Процедура остановки KESL доступна break-glass admin.
-- [ ] Процедура `apt-get purge kesl` проверена по документации текущей версии.
+- [x] Процедура `apt-get purge kesl` проверена по документации текущей версии.
-- [ ] Откат не использует `docker compose down -v` и не удаляет volumes.
+- [x] Откат не использует `docker compose down -v` и не удаляет volumes.
-- [ ] После отката предусмотрены smoke, health и firewall checks.
+- [x] После отката предусмотрены smoke, health и firewall checks.
-- [ ] Reboot выполняется только отдельным согласованным окном при необходимости.
+- [x] Reboot выполняется только отдельным согласованным окном при необходимости.
 
-Результат rehearsal/desk check:
+Результат rehearsal/desk check: полный uninstall в этом окне не выполнялся;
+рабочий откат FTP — `--stop-task 1`.
 
 ## 8. Итоговая приёмка
 
-- АВЗ.1: принято / не принято.
+- АВЗ.1: **принято для тестовой ВМ**, с оговоркой on-access.
-- АВЗ.2: принято / не принято.
+- АВЗ.2: **принято для тестовой ВМ**, с оговоркой ненаблюдавшегося hourly
+  цикла и отсутствия alert.
 - Ограничения/остаточные риски:
-- Следующий review:
+  - 4 ГБ RAM, swap ~675 МБ; профиль `ScanMemoryLimit=512` /
+    `MaxMemory=1024MB` **не переносить** в production;
+  - коммерческая, не сертифицированная ФСТЭК сборка;
+  - KSN выключен;
+  - сетевые компоненты KESL остановлены сознательно;
+  - container monitoring недоступен по лицензии;
+  - OAS не удалил EICAR в момент записи; реакция через ODS;
+  - внешний smoke и 24h наблюдение не закрыты;
+  - публичный SSH 22 остаётся открытым (не KESL).
+- Следующий review: перед боевым внедрением после увеличения RAM ≥16 ГБ.
 - Operations, ФИО/подпись/дата:
 - Security, ФИО/подпись/дата:
 - Service owner, ФИО/подпись/дата:

VM1_app/codebase/backend/deployment/kesl/RUNBOOK.KESL.ru.md

@@ -69,8 +69,8 @@
 - `UseOnDemandCPULimit=Yes`, `OnDemandCPULimit=15`;
 - не запускать full filesystem scan и проверку архивов;
 - ODS и ContainerScan выполнять по одному объекту, не одновременно;
-- сначала Update и health, затем один stateless image, затем File Threat
+- сначала Update и health, затем один ограниченный `scan-file`, затем File Threat
-  Protection в `Notify`;
+  Protection; `InterceptorProtectionMode` на fanotify недоступен;
 - остановить KESL при available memory менее 512 МБ, устойчивом swap IO,
   появлении host/container OOM, restart или провале smoke;
 - до режима `Block` требуется отдельное подтверждение стабильности.
@@ -169,18 +169,85 @@ journalctl --since '-30 min' --no-pager \
 
 ## 2. Проверка пакета и подготовка
 
-GUI на сервер не устанавливается. Пакет передаётся в root-only staging,
+GUI на сервер не устанавливается. Дистрибутив и DEB копируются в root-only
-например `/root/kesl-install`, и удаляется после приёмки.
+staging `/root/kesl-install` и удаляются после приёмки. ISO в
+`/var/lib/han-deploy/incoming` не распаковывается на месте и не остаётся
+смонтированным после извлечения пакета.
 
-```sh
+Для тестовой ВМ ожидаемый ISO: `049-16-d-01.iso`, volume id `KESL12SP4`.
-install -d -m 0700 -o root -g root /root/kesl-install
+
-install -m 0600 -o root -g root \
+Вендор публикует контрольную сумму **ФИКС 2.0.2 / ГОСТ Р 34.11-94, программно**,
-  /tmp/<KESL_12_4_AMD64_DEB> /root/kesl-install/kesl.deb
+а не SHA-256. Это разные алгоритмы: оба дают 64 hex-символа, поэтому
-sha256sum /root/kesl-install/kesl.deb
+`sha256sum --check` против суммы с сайта закономерно не совпадает.
-dpkg-deb -f /root/kesl-install/kesl.deb Package Version Architecture
+
+Ожидаемая сумма вендора (ГОСТ Р 34.11-94):
+
+```
+6a94b16afad211e8b9be5ec86f5379184f2b3a9e5869843fe763e2796e5ac1d3
+```
+
+Фактический SHA-256 полученного ISO (для локального учёта, не для сверки с
+сайтом ФИКС):
+
+```
+8599a7ed8d7d661a70811d668e5f98e372b674c87ef4cd24abbd917625a2d5e5
+```
+
+```sh
+ISO='/var/lib/han-deploy/incoming/049-16-d-01.iso'
+SIG='/var/lib/han-deploy/incoming/049-16-d-01.sig'
+
+ls -l "$ISO" "$SIG"
+file "$ISO" "$SIG"
+sha256sum "$ISO"
+command -v rhash || true
+openssl list -digest-algorithms 2>/dev/null | grep -i gost || true
+```
+
+Если доступен `rhash`, посчитайте ГОСТ и сравните с суммой вендора. ФИКС
+может использовать другой набор S-блоков/порядок байт, поэтому проверьте
+несколько вариантов:
+
+```sh
+rhash --gost "$ISO"
+rhash --gost --gost-reverse "$ISO"
+rhash --gost-cryptopro "$ISO"
+rhash --gost-cryptopro --gost-reverse "$ISO"
+```
+
+Для ISO `049-16-d-01.iso` сумма вендора совпала с `rhash --gost`.
+
+Подлинность принимается, если совпала ГОСТ-сумма **или** одновременно
+выполнены все условия ниже:
+
+- volume id ISO = `KESL12SP4`;
+- внутри есть `kesl/kesl_12.4.0-1225_amd64.deb`;
+- SHA-256 ISO зафиксирован в evidence;
+- источник — официальный канал вендора.
+
+Файл `.sig` без доверенного публичного ключа не заменяет эту проверку.
+
+Только после совпадения хеша смонтируйте ISO только для чтения и найдите
+`kesl_*_amd64.deb` без GUI:
+
+```sh
+install -d -m 0700 -o root -g root /root/kesl-install /mnt/kesl-iso
+mount -o ro,loop "$ISO" /mnt/kesl-iso
+find /mnt/kesl-iso -type f \( -iname '*kesl*' -o -iname '*.deb' \) | sort
+```
+
+Ожидается имя вида `kesl_12.4.0-*_amd64.deb`. Не копируйте `kesl-gui`, i386,
+arm64, RPM, KSC agent и Windows-пакеты. Если внутри только 12.0.x или нет
+amd64 DEB — stop: для Ubuntu 24.04 нужен KESL 12.4.
+
+```sh
+install -m 0600 -o root -g root \
+  <KESL_12_4_AMD64_DEB_FROM_ISO> /root/kesl-install/kesl.deb
+dpkg-deb -f /root/kesl-install/kesl.deb Package Version Architecture
+umount /mnt/kesl-iso
+rmdir /mnt/kesl-iso
 ```
 
-Сравните SHA-256 с опубликованным/полученным по доверенному каналу значением.
 Не продолжайте при package name не `kesl`, неверной архитектуре или версии не
 12.4.x.
 
@@ -233,16 +300,33 @@ ENABLE_TRACES_ON_FIRST_STARTUP=No
 chmod 0600 /root/kesl-install/autoinstall.ini
 /opt/kaspersky/kesl/bin/kesl-setup.pl \
   --autoinstall=/root/kesl-install/autoinstall.ini
-test "$?" -eq 0
+echo "kesl-setup exit=$?"
+# Exit 71 with INSTALL_LICENSE=None is expected: setup treats None as a
+# code. Do not rerun setup. Continue if kesl.service is active and
+# File_Threat_Protection is Stopped.
 systemctl --no-pager status kesl
 kesl-control --app-info
 kesl-control --supported-tech-info
 kesl-control --get-task-list
 ```
 
-Если используется activation code, не передавайте его аргументом команды и не
+Активацию кодом выполняйте только после успешного `kesl-setup.pl` и до
-храните в этом репозитории. Выполните активацию по официальной инструкции
+загрузки баз. Код не помещайте в `autoinstall.ini`, репозиторий, evidence,
-Kaspersky с защищённым локальным вводом/файлом ключа.
+чат и историю shell:
+
+```sh
+set +o history
+unset HISTFILE
+read -rsp 'KESL activation code: ' KESL_CODE; echo
+kesl-control --add-active-key "$KESL_CODE"
+unset KESL_CODE
+set -o history
+kesl-control -L --query
+kesl-control --app-info
+```
+
+Нужен исходящий доступ к серверам активации Kaspersky. В выводе лицензии
+оставьте только статус «ключ действителен» и срок; сам код не копируйте.
 
 ## 4. Ресурсные ограничения до первого scan
 
@@ -306,13 +390,19 @@ kesl-control --app-info
 выпуска баз и успешное завершение Update. Затем задайте почасовой запуск:
 
 ```sh
-kesl-control --set-schedule 6 RuleType=Hourly
+START="$(LC_ALL=C date +'%Y/%b/%d %H:%M:%S;1')"
+kesl-control --set-schedule 6 \
+  RuleType=Hourly \
+  "StartTime=${START}" \
+  RunMissedStartRules=No \
+  RandomInterval=0
 kesl-control --get-schedule 6
 ```
 
-Если установленная сборка требует интервал в `StartTime`, не угадывайте
+Для 12.4 одного `RuleType=Hourly` недостаточно: нужен `StartTime` вида
-синтаксис: экспортируйте schedule и измените его по документации именно этой
+`2026/Sep/07 12:00:00;1` (английское имя месяца, интервал 1 час). `LC_ALL=C`
-сборки. До успешного автообновления АВЗ.2 не принимается.
+обязателен, иначе локаль `ru_RU` подставит русское имя месяца. До успешного
+автообновления АВЗ.2 не принимается.
 
 Сразу после обновления повторите разделы 1.2–1.3. При деградации выполните
 rollback из раздела 11.
@@ -422,39 +512,33 @@ kesl-control --get-settings 1
 kesl-control --get-task-state 1
 ```
 
-Для пилота включите асинхронный режим перехватчика `Notify`, при котором
+Для пилота запустите File Threat Protection. На Ubuntu 24.04 с `fanotify`
-KESL журналирует обнаружения, но не выполняет блокирующее действие:
+параметр `InterceptorProtectionMode` **не поддерживается** (`Unsupported
+setting`): перехватчик работает в штатном блокирующем режиме на время
+проверки. Отдельный `Notify` недоступен; откат — остановка задачи 1.
 
 ```sh
-kesl-control --set-app-settings InterceptorProtectionMode=Notify
-kesl-control --start-task 1
-kesl-control --get-task-state 1
-```
-
-Пилот длится минимум 2–4 часа обычной нагрузки. Каждые 15 минут проверяйте
-метрики раздела 1 и события KESL. Не считайте этот режим конечной реализацией
-АВЗ.1: он не обеспечивает блокирование/лечение.
-
-Если пилот стабилен, в том же maintenance window:
-
-1. проверьте, что `ActionOnThreat=DisinfectDeleteIfNotPossible`;
-2. установите `InterceptorProtectionMode=Block`;
-3. перезапустите task 1, если этого требует текущая сборка;
-4. повторите health, smoke, firewall и resource checks.
-
-```sh
-kesl-control --set-settings 1 \
-  ActionOnThreat=DisinfectDeleteIfNotPossible ScanArchived=No
-kesl-control --set-app-settings InterceptorProtectionMode=Block
-kesl-control --stop-task 1
 kesl-control --start-task 1
 kesl-control --get-task-state 1
 kesl-control --app-info
 ```
 
-В режиме `Block` доступ к файлу ожидает результат проверки. При появлении
+Пилот длится минимум 15–30 минут на constrained test VM и 2–4 часа на
-latency вернитесь в `Notify` либо остановите task 1 и выполните rollback;
+боевых ресурсах. Каждые 15 минут проверяйте метрики раздела 1, события KESL
-не расширяйте исключения вслепую.
+и firewall. АВЗ.1 выполняется при `Started` + `ActionOnThreat=DisinfectDeleteIfNotPossible`.
+
+Перед приёмкой:
+
+```sh
+kesl-control --set-settings 1 \
+  ActionOnThreat=DisinfectDeleteIfNotPossible ScanArchived=No
+kesl-control --get-task-state 1
+kesl-control --app-info
+```
+
+При latency/swap/unhealthy выполните `kesl-control --stop-task 1`. Не
+расширяйте исключения вслепую. Network Threat Protection, Firewall
+Management, Web Threat Protection и Behavior Detection не запускайте.
 
 ## 9. Приёмочный тест и evidence
 
@@ -483,8 +567,15 @@ kesl-control --get-schedule 6
 kesl-control -E --query -n 100 --reverse
 ```
 
-Очистите тестовый каталог после подтверждения реакции и заполните
+Очистите тестовый каталог после подтверждения реакции. Копию EICAR в Backup
-`deployment/kesl/EVIDENCE.AVZ.ru.md`.
+удалите точечно, не весь Backup:
+
+```sh
+kesl-control -B --query --reverse -n 20
+kesl-control -B --mass-remove --query "DetectName == 'EICAR-Test-File'"
+```
+
+Заполните `deployment/kesl/EVIDENCE.AVZ.ru.md`.
 
 ## 10. Финальные проверки
 
@@ -535,7 +626,7 @@ ufw status verbose
 Сначала минимально обратимое действие:
 
 ```sh
-kesl-control --set-app-settings InterceptorProtectionMode=Notify
+kesl-control --stop-task 1
 ```
 
 Если управление KESL не отвечает:

VM1_app/codebase/backend/deployment/observability/otel-host-collector.yaml

@@ -0,0 +1,143 @@
+extensions:
+  file_storage:
+    directory: /var/lib/han-otel/host-collector
+    timeout: 10s
+
+receivers:
+  filelog/docker:
+    include: [/var/lib/docker/containers/*/*-json.log]
+    start_at: end
+    include_file_path: true
+    storage: file_storage
+    operators:
+      - type: json_parser
+        id: docker-json
+        timestamp:
+          parse_from: attributes.time
+          layout_type: gotime
+          layout: "2006-01-02T15:04:05.000000000Z07:00"
+      - type: regex_parser
+        id: docker-service-label
+        parse_from: attributes["attrs"]["tag"]
+        if: 'attributes["attrs"]["tag"] != nil'
+        regex: '^(?P<docker_service>nginx|keycloak|redis)$'
+      - type: move
+        from: attributes.docker_service
+        to: resource["service.name"]
+      - type: move
+        from: attributes.stream
+        to: attributes["log.iostream"]
+      - type: move
+        from: attributes.log
+        to: body
+      - type: json_parser
+        id: structured-body
+        parse_from: body
+        parse_to: attributes
+        if: 'body matches "^\\s*\\{"'
+        on_error: send
+  journald/host:
+    directory: /var/log/journal
+    units:
+      - docker.service
+      - han-stack@production.service
+      - han-secrets@production.service
+      - han-chat-docker-firewall.service
+      - certbot.service
+      - fail2ban.service
+    priority: info
+    start_at: end
+    storage: file_storage
+
+processors:
+  memory_limiter:
+    check_interval: 1s
+    limit_mib: 192
+    spike_limit_mib: 48
+  resource/common:
+    attributes:
+      - {key: service.namespace, value: han-chat, action: upsert}
+      - {key: deployment.environment, value: "${env:APP_ENV}", action: upsert}
+      - {key: service.version, value: "${env:RELEASE_VERSION}", action: upsert}
+      - {key: host.name, value: "${env:HOST_NAME}", action: upsert}
+  transform/journal:
+    error_mode: ignore
+    log_statements:
+      - context: log
+        statements:
+          - set(resource.attributes["service.name"], attributes["_SYSTEMD_UNIT"]) where resource.attributes["service.name"] == nil and attributes["_SYSTEMD_UNIT"] != nil
+          - set(attributes["event.source"], "journald") where attributes["_SYSTEMD_UNIT"] != nil
+          - set(attributes["event.source"], "docker-json-file") where attributes["_SYSTEMD_UNIT"] == nil
+          - set(attributes["route.class"], "api") where resource.attributes["service.name"] == "nginx" and IsMatch(attributes["uri"], "^/api/")
+          - set(attributes["route.class"], "auth") where resource.attributes["service.name"] == "nginx" and IsMatch(attributes["uri"], "^/auth/")
+          - set(attributes["route.class"], "bitrix") where resource.attributes["service.name"] == "nginx" and IsMatch(attributes["uri"], "^/bitrix/")
+          - set(attributes["route.class"], "callbacks") where resource.attributes["service.name"] == "nginx" and IsMatch(attributes["uri"], "^/callbacks/")
+          - set(attributes["route.class"], "static") where resource.attributes["service.name"] == "nginx" and attributes["route.class"] == nil
+          - replace_pattern(body, "(?i)(Bearer|Basic)\\s+[A-Za-z0-9._~+/=-]+", "$$1 [REDACTED]") where IsString(body)
+          - replace_pattern(body, "(?i)(token|secret|password|code)=([^&\\s]+)", "$$1=[REDACTED]") where IsString(body)
+          - set(body, "nginx.access") where resource.attributes["service.name"] == "nginx" and attributes["log.iostream"] == "stdout"
+          - set(body, "keycloak.event") where resource.attributes["service.name"] == "keycloak" and attributes["log.iostream"] == "stdout"
+  attributes/redact:
+    actions:
+      - {key: authorization, action: delete}
+      - {key: http.request.header.authorization, action: delete}
+      - {key: http.request.header.cookie, action: delete}
+      - {key: http.response.header.set-cookie, action: delete}
+      - {key: url.query, action: delete}
+      - {key: url.full, action: delete}
+      - {key: http.target, action: delete}
+      - {key: http.request.body, action: delete}
+      - {key: db.statement, action: delete}
+      - {key: message, action: delete}
+      - {key: payload, action: delete}
+      - {key: user.phone, action: delete}
+      - {key: user.email, action: delete}
+      - {key: remote_addr, action: delete}
+      - {key: user_agent, action: delete}
+      - {key: uri, action: delete}
+  filter/allowlist:
+    error_mode: ignore
+    logs:
+      log_record:
+        - 'attributes["_SYSTEMD_UNIT"] == nil and resource.attributes["service.name"] != "nginx" and resource.attributes["service.name"] != "keycloak" and resource.attributes["service.name"] != "redis"'
+        - 'resource.attributes["service.name"] == "otel-collector" or resource.attributes["service.name"] == "otel-host-collector"'
+  filter/noise:
+    error_mode: ignore
+    logs:
+      log_record:
+        - 'resource.attributes["service.name"] == "nginx" and severity_number < SEVERITY_NUMBER_WARN and IsMatch(body, "(/health/live|/nginx-health/live|/.well-known/acme-challenge/)")'
+  batch:
+    timeout: 5s
+    send_batch_size: 512
+    send_batch_max_size: 1024
+
+exporters:
+  otlp/remote:
+    endpoint: "${env:OTEL_REMOTE_ENDPOINT}"
+    headers:
+      authorization: ${file:/run/han-chat/secrets/OTEL_REMOTE_AUTH_HEADER}
+    tls:
+      insecure: "${env:OTEL_REMOTE_TLS_INSECURE}"
+    sending_queue:
+      enabled: true
+      queue_size: 5000
+      storage: file_storage
+    retry_on_failure:
+      enabled: true
+      initial_interval: 5s
+      max_interval: 30s
+      max_elapsed_time: 0s
+
+service:
+  extensions: [file_storage]
+  pipelines:
+    logs:
+      receivers: [filelog/docker, journald/host]
+      processors:
+        [memory_limiter, resource/common, filter/allowlist, filter/noise, transform/journal, attributes/redact, batch]
+      exporters: [otlp/remote]
+  telemetry:
+    logs:
+      level: error
+    metrics:
+      level: none

VM1_app/codebase/backend/deployment/preflight.sh

@@ -19,6 +19,7 @@ value() {
 [ -f "$ROOT/docker-compose.yml" ] || fail "root docker-compose.yml is missing"
 [ -f "$ENV_FILE" ] || fail ".env is missing"
 [ -f "$MANIFEST" ] || fail "runtime secret manifest is missing"
+[ -d /var/log/journal ] || fail "persistent journald directory is missing"
 [ -L /opt/han-chat/current ] || fail "/opt/han-chat/current must be a root-controlled release link"
 [ "$(/usr/bin/stat -c '%U:%G' /opt/han-chat/current)" = root:root ] ||
   fail "active release link must be root:root"
@@ -59,6 +60,8 @@ if [ -d "$ROOT" ]; then
 $ROOT/docker-compose.yml
 $ROOT/deployment/preflight.sh
 $ROOT/deployment/han-stack@.service
+$ROOT/deployment/han-host-otel-collector@.service
+$ROOT/deployment/observability/otel-host-collector.yaml
 $ROOT/deployment/scripts/tls-deploy-hook.sh
 $ROOT/deployment/secrets/han-compose
 $ROOT/deployment/secrets/han-secrets
@@ -91,6 +94,10 @@ if [ -f "$ENV_FILE" ]; then
   [ "$(value NGINX_TLS_ENABLED)" = true ] || fail "NGINX_TLS_ENABLED must be true"
   [ "$(value NGINX_HTTP_PORT)" = 80 ] || fail "nginx must publish host port 80"
   [ "$(value NGINX_HTTPS_PORT)" = 443 ] || fail "nginx must publish host port 443"
+  [ -n "$(value HOST_NAME)" ] || fail "HOST_NAME is required for telemetry correlation"
+  echo "$(value TELEMETRYGEN_IMAGE)" |
+    grep -Eq '^ghcr\.io/.+@sha256:[a-f0-9]{64}$' ||
+    fail "TELEMETRYGEN_IMAGE must be pinned by digest"
   [ "$(value NGINX_TLS_CERTIFICATE)" = /run/tls/fullchain.pem ] ||
     fail "nginx certificate must use staged /run/tls/fullchain.pem"
   [ "$(value NGINX_TLS_CERTIFICATE_KEY)" = /run/tls/privkey.pem ] ||
@@ -278,6 +285,39 @@ if [ -f "$MANIFEST" ]; then
   IFS=$old_ifs
 fi
 
+host_otel_config="$ROOT/deployment/observability/otel-host-collector.yaml"
+if [ -f "$host_otel_config" ]; then
+  grep -Fq 'filelog/docker' "$host_otel_config" ||
+    fail "host collector must include Docker filelog"
+  grep -Fq 'journald/host' "$host_otel_config" ||
+    fail "host collector must include journald allow-list"
+  grep -Fq 'filter/allowlist' "$host_otel_config" ||
+    fail "host collector must drop non-allowlisted containers"
+  grep -Fq 'file_storage' "$host_otel_config" ||
+    fail "host collector must persist offsets and exporter queue"
+  ! grep -Fq '/var/run/docker.sock' "$host_otel_config" ||
+    fail "Docker socket access is forbidden"
+fi
+[ -x /usr/local/bin/otelcol-contrib ] ||
+  fail "pinned host otelcol-contrib is not installed"
+otel_checksum=/usr/local/share/han-otel/otelcol-contrib.sha256
+otel_version=/usr/local/share/han-otel/otelcol-contrib.version
+[ -f "$otel_checksum" ] || fail "host Collector checksum record is missing"
+[ -f "$otel_version" ] || fail "host Collector version record is missing"
+if [ -f "$otel_checksum" ]; then
+  (cd / && sha256sum -c "$otel_checksum") ||
+    fail "host Collector binary checksum mismatch"
+fi
+if [ -x /usr/local/bin/otelcol-contrib ] && [ -f "$otel_version" ]; then
+  /usr/local/bin/otelcol-contrib --version 2>&1 |
+    grep -Fq "$(cat "$otel_version")" ||
+    fail "host Collector binary version mismatch"
+fi
+if [ -x /usr/local/bin/otelcol-contrib ] && [ -f "$host_otel_config" ]; then
+  /usr/local/bin/otelcol-contrib validate --config="$host_otel_config" ||
+    fail "host collector config validation failed"
+fi
+
 if [ -x "$ROOT/scripts/validate-env" ] && [ -f "$ENV_FILE" ] && [ -f "$MANIFEST" ]; then
   "$ROOT/scripts/validate-env" "$ENV_FILE" --runtime-manifest "$MANIFEST" ||
     fail "config/runtime validator rejected the production inputs"

VM1_app/codebase/backend/deployment/scripts/setup-vm.sh

@@ -27,6 +27,8 @@ HARDEN_SSH="${HARDEN_SSH:-false}"
 LOCK_ACCOUNT_PASSWORDS="${LOCK_ACCOUNT_PASSWORDS:-true}"
 RESET_UFW="${RESET_UFW:-true}"
 SKIP_APT_UPGRADE="${SKIP_APT_UPGRADE:-false}"
+OTEL_HOST_COLLECTOR_VERSION="${OTEL_HOST_COLLECTOR_VERSION:-}"
+OTEL_HOST_COLLECTOR_SHA256="${OTEL_HOST_COLLECTOR_SHA256:-}"
 LOG_FILE="${LOG_FILE:-/var/log/han-chat-vm1-setup.log}"
 
 log() {
@@ -120,6 +122,13 @@ configure_time() {
   timedatectl set-ntp true
 }
 
+configure_journald() {
+  step "Persistent journald для host telemetry"
+  install -d -m 2755 -o root -g systemd-journal /var/log/journal
+  systemd-tmpfiles --create --prefix /var/log/journal
+  journalctl --flush
+}
+
 install_authorized_key() {
   local user=$1 source=$2 target="/home/${1}/.ssh/authorized_keys"
   [[ -f "$source" && ! -L "$source" ]] || die "Не найден обычный key file ${source}"
@@ -422,7 +431,7 @@ install_deploy_sudoers() {
   step "Exact sudoers для deploy"
   cat >/etc/sudoers.d/deploy <<'EOF'
 Cmnd_Alias HAN_VM1_UNITS = /usr/bin/systemctl start han-secrets@production.service, /usr/bin/systemctl restart han-secrets@production.service, /usr/bin/systemctl start han-stack@production.service, /usr/bin/systemctl restart han-stack@production.service, /usr/bin/systemctl stop han-stack@production.service
-Cmnd_Alias HAN_VM1_STATUS = /usr/bin/systemctl --no-pager status han-secrets@production.service, /usr/bin/systemctl --no-pager status han-stack@production.service, /usr/bin/journalctl --no-pager -u han-secrets@production.service, /usr/bin/journalctl --no-pager -u han-stack@production.service
+Cmnd_Alias HAN_VM1_STATUS = /usr/bin/systemctl --no-pager status han-secrets@production.service, /usr/bin/systemctl --no-pager status han-stack@production.service, /usr/bin/systemctl --no-pager status han-host-otel-collector@production.service, /usr/bin/journalctl --no-pager -u han-secrets@production.service, /usr/bin/journalctl --no-pager -u han-stack@production.service, /usr/bin/journalctl --no-pager -u han-host-otel-collector@production.service
 deploy ALL=(root) NOPASSWD: HAN_VM1_UNITS, HAN_VM1_STATUS
 EOF
   chmod 0440 /etc/sudoers.d/deploy
@@ -447,6 +456,15 @@ install_release_helpers_if_possible() {
   [[ -x "${deployment}/scripts/tls-deploy-hook.sh" ]] || die "TLS hook не executable"
   [[ -x "${source_dir}/han-compose" ]] || die "han-compose не executable"
   [[ -x "${source_dir}/han-secrets" ]] || die "han-secrets не executable"
+  [[ -f "${deployment}/han-host-otel-collector@.service" ]] \
+    || die "han-host-otel-collector@.service отсутствует"
+  [[ -f "${deployment}/observability/otel-host-collector.yaml" ]] \
+    || die "otel-host-collector.yaml отсутствует"
+  [[ "$OTEL_HOST_COLLECTOR_VERSION" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]] \
+    || die "Задайте OTEL_HOST_COLLECTOR_VERSION=x.y.z"
+  [[ "$OTEL_HOST_COLLECTOR_SHA256" =~ ^[a-f0-9]{64}$ ]] \
+    || die "Задайте проверенный OTEL_HOST_COLLECTOR_SHA256"
+  [[ "$(uname -m)" == x86_64 ]] || die "Host Collector artifact рассчитан на amd64"
   if getent group "$tls_group" >/dev/null; then
     [[ "$(getent group "$tls_group" | cut -d: -f3)" == "$tls_gid" ]] \
       || die "han-nginx-tls имеет неожиданный GID"
@@ -472,6 +490,27 @@ install_release_helpers_if_possible() {
   install -m 0644 -o root -g root \
     "${deployment}/han-stack@.service" \
     /etc/systemd/system/han-stack@.service
+  local otel_archive otel_tmp otel_url
+  otel_archive="$(mktemp)"
+  otel_tmp="$(mktemp -d)"
+  otel_url="https://github.com/open-telemetry/opentelemetry-collector-releases/releases/download/v${OTEL_HOST_COLLECTOR_VERSION}/otelcol-contrib_${OTEL_HOST_COLLECTOR_VERSION}_linux_amd64.tar.gz"
+  curl --fail --location --proto '=https' --tlsv1.2 \
+    --output "$otel_archive" "$otel_url"
+  printf '%s  %s\n' "$OTEL_HOST_COLLECTOR_SHA256" "$otel_archive" | sha256sum -c -
+  tar -xzf "$otel_archive" -C "$otel_tmp" otelcol-contrib
+  install -m 0755 -o root -g root "$otel_tmp/otelcol-contrib" /usr/local/bin/otelcol-contrib
+  install -d -m 0755 -o root -g root /usr/local/share/han-otel
+  sha256sum /usr/local/bin/otelcol-contrib \
+    >/usr/local/share/han-otel/otelcol-contrib.sha256
+  printf '%s\n' "$OTEL_HOST_COLLECTOR_VERSION" \
+    >/usr/local/share/han-otel/otelcol-contrib.version
+  chmod 0644 /usr/local/share/han-otel/otelcol-contrib.sha256 \
+    /usr/local/share/han-otel/otelcol-contrib.version
+  rm -rf "$otel_archive" "$otel_tmp"
+  install -d -m 0700 -o root -g root /var/lib/han-otel/host-collector
+  install -m 0644 -o root -g root \
+    "${deployment}/han-host-otel-collector@.service" \
+    /etc/systemd/system/han-host-otel-collector@.service
   install -d -m 0755 -o root -g root /etc/letsencrypt/renewal-hooks/deploy
   install -m 0755 -o root -g root \
     "${deployment}/scripts/tls-deploy-hook.sh" \
@@ -481,6 +520,7 @@ install_release_helpers_if_possible() {
     /usr/local/share/doc/han-secrets/SELECTEL_RUNBOOK.ru.md
   systemctl daemon-reload
   systemctl enable certbot.timer
+  systemctl enable han-host-otel-collector@production.service
   log "Helpers установлены; application units не запущены"
 }
 
@@ -539,6 +579,7 @@ main() {
   check_os
   update_system
   configure_time
+  configure_journald
   create_host_roles
   configure_account_passwords
   configure_layout

VM1_app/codebase/backend/deployment/scripts/verify-observability.sh

@@ -16,11 +16,38 @@ compose() { docker compose --env-file "$CONFIG_FILE" "$@"; }
 
 NETWORK="${OBSERVABILITY_NETWORK:-han-chat-observability}"
 COLLECTOR_SERVICE="${COLLECTOR_SERVICE:-otel-collector}"
+TELEMETRYGEN_IMAGE="${TELEMETRYGEN_IMAGE:-}"
 errors=0
 
 ok() { printf 'OK   %s\n' "$*"; }
 fail() { printf 'FAIL %s\n' "$*" >&2; errors=$((errors + 1)); }
 
+if systemctl is-active --quiet han-host-otel-collector@production.service; then
+  ok "Host Collector service is running"
+else
+  fail "Host Collector service is not running"
+fi
+
+if /usr/local/bin/otelcol-contrib validate \
+  --config=deployment/observability/otel-host-collector.yaml >/dev/null 2>&1; then
+  ok "Host Collector configuration is valid"
+else
+  fail "Host Collector configuration is invalid"
+fi
+
+host_bad_logs="$(
+  journalctl --since=-10min --no-pager \
+    -u han-host-otel-collector@production.service 2>&1 |
+    grep -Ei 'queue is full|permission denied|connection refused|tls:|Unauthenticated|Permanent error' ||
+    true
+)"
+if [[ -z "$host_bad_logs" ]]; then
+  ok "No host Collector errors in last 10 minutes"
+else
+  fail "Host Collector reports read/export/queue errors"
+  printf '%s\n' "$host_bad_logs" >&2
+fi
+
 collector_id="$(compose ps -q "$COLLECTOR_SERVICE" 2>/dev/null || true)"
 if [[ -n "$collector_id" ]] &&
   [[ "$(docker inspect --format '{{.State.Status}}' "$collector_id")" == running ]]; then
@@ -43,12 +70,16 @@ PY
   fi
 done
 
-docker run --rm --network "$NETWORK" \
+if [[ ! "$TELEMETRYGEN_IMAGE" =~ @sha256:[a-f0-9]{64}$ ]]; then
-  ghcr.io/open-telemetry/opentelemetry-collector-contrib/telemetrygen:latest \
+  fail "TELEMETRYGEN_IMAGE must be pinned by sha256 digest"
+elif docker run --rm --network "$NETWORK" \
+  "$TELEMETRYGEN_IMAGE" \
   traces --otlp-endpoint otel-collector:4317 --otlp-insecure \
-  --service han-chat-e2e-canary --traces 100 --rate 20 >/dev/null \
+  --service han-chat-e2e-canary --traces 100 --rate 20 >/dev/null; then
-  && ok "100 canary traces submitted" \
+  ok "100 canary traces submitted"
-  || fail "telemetrygen failed"
+else
+  fail "telemetrygen failed"
+fi
 
 bad_logs="$(
   compose logs --since=10m "$COLLECTOR_SERVICE" 2>&1 |
@@ -72,4 +103,12 @@ cat <<'EOF'
   service.namespace = han-chat
 Затем выполните synthetic API request и проверьте общий trace между
 api-backend и dependency spans, service.version и deployment.environment.
+В Logs проверьте:
+  host.name = <VM1 HOST_NAME>
+  service.name IN (nginx, keycloak, redis, api-backend, sms-service,
+                   sms-worker, bitrix-local-app)
+У canary application log trace_id/span_id должны открывать соответствующий
+span. Python events должны встречаться один раз, unknown-container и
+otel-host-collector отсутствовать. Fake token/PII marker не должен находиться
+ни в logs, ни в traces.
 EOF

VM1_app/codebase/backend/infra/compose/application.yml

@@ -29,7 +29,7 @@ x-api-secrets: &api-secrets
 
 x-api-runtime: &api-runtime
   image: ${API_BACKEND_IMAGE:?API_BACKEND_IMAGE must be pinned by digest}
-  environment:
+  environment: &api-environment
     <<: *api-secret-environment
     APP_ENV: ${APP_ENV:-production-like}
     LOG_LEVEL: ${LOG_LEVEL:-INFO}
@@ -167,6 +167,8 @@ services:
       KC_HTTP_RELATIVE_PATH: /auth
       KC_HEALTH_ENABLED: "true"
       KC_METRICS_ENABLED: "true"
+      KC_LOG_CONSOLE_OUTPUT: json
+      KC_LOG_LEVEL: ${KEYCLOAK_LOG_LEVEL:-info}
       KC_HOSTNAME: ${KEYCLOAK_PUBLIC_URL}
       PUBLIC_WEB_URL: ${PUBLIC_WEB_URL:?PUBLIC_WEB_URL is required for realm import}
       KC_BOOTSTRAP_ADMIN_USERNAME: ${KEYCLOAK_ADMIN}
@@ -185,6 +187,8 @@ services:
       - keycloak_settings_bridge_token
       - keycloak_sms_service_token
     command: ["start", "--optimized", "--import-realm"]
+    labels:
+      com.han.service.name: keycloak
     expose: ["8080", "9000"]
     volumes:
       - type: bind
@@ -209,7 +213,10 @@ services:
       core: {soft: 0, hard: 0}
     logging:
       driver: json-file
-      options: {max-size: "50m", max-file: "5"}
+      options:
+        max-size: "50m"
+        max-file: "5"
+        tag: keycloak
 
   sms-service:
     <<: *sms-runtime
@@ -269,6 +276,9 @@ services:
   delivery-worker:
     <<: *api-runtime
     command: ["han-delivery-worker"]
+    environment:
+      <<: *api-environment
+      OTEL_SERVICE_NAME: delivery-worker
     depends_on:
       api-backend: {condition: service_healthy}
       bitrix-local-app: {condition: service_healthy}
@@ -283,6 +293,9 @@ services:
   safety-recovery-worker:
     <<: *api-runtime
     command: ["han-safety-worker"]
+    environment:
+      <<: *api-environment
+      OTEL_SERVICE_NAME: safety-recovery-worker
     depends_on:
       api-backend: {condition: service_healthy}
     healthcheck:
@@ -296,6 +309,9 @@ services:
   cleanup-worker:
     <<: *api-runtime
     command: ["han-cleanup-worker"]
+    environment:
+      <<: *api-environment
+      OTEL_SERVICE_NAME: cleanup-worker
     depends_on:
       api-backend: {condition: service_healthy}
     healthcheck:
@@ -309,6 +325,9 @@ services:
   notification-expire-worker:
     <<: *api-runtime
     command: ["han-notification-expire-worker"]
+    environment:
+      <<: *api-environment
+      OTEL_SERVICE_NAME: notification-expire-worker
     depends_on:
       api-backend: {condition: service_healthy}
     healthcheck:
@@ -322,6 +341,9 @@ services:
   notification-draft-cleanup-worker:
     <<: *api-runtime
     command: ["han-notification-draft-cleanup-worker"]
+    environment:
+      <<: *api-environment
+      OTEL_SERVICE_NAME: notification-draft-cleanup-worker
     depends_on:
       api-backend: {condition: service_healthy}
     healthcheck:
@@ -346,6 +368,10 @@ services:
       BITRIX_API_FORWARD_TOKEN_FILE: /run/secrets/bitrix_api_forward_token
       BITRIX_TOKEN_ENCRYPTION_KEY_FILE: /run/secrets/bitrix_token_encryption_key
       APP_ENV: ${APP_ENV:-production-like}
+      LOG_LEVEL: ${LOG_LEVEL:-INFO}
+      RELEASE_VERSION: ${RELEASE_VERSION:-unknown}
+      OTEL_EXPORTER_OTLP_ENDPOINT: ${OTEL_EXPORTER_OTLP_ENDPOINT:-http://otel-collector:4317}
+      OTEL_SERVICE_NAME: bitrix-local-app
       BITRIX_CLIENT_ID: ${BITRIX_CLIENT_ID}
       BITRIX_CONNECTOR_ID: ${BITRIX_CONNECTOR_ID:-han_mobile_app}
       BITRIX_CONNECTOR_NAME: ${BITRIX_CONNECTOR_NAME:-HAN Mobile App}

VM1_app/codebase/backend/nginx/docker-compose.yml

@@ -1,6 +1,8 @@
 services:
   nginx:
     image: ${NGINX_IMAGE:?NGINX_IMAGE must be pinned by digest}
+    labels:
+      com.han.service.name: nginx
     environment:
       APP_ENV: ${APP_ENV:-production-like}
       PUBLIC_HOST: ${PUBLIC_HOST}
@@ -64,4 +66,7 @@ services:
       nofile: {soft: 65536, hard: 65536}
     logging:
       driver: json-file
-      options: {max-size: "50m", max-file: "5"}
+      options:
+        max-size: "50m"
+        max-file: "5"
+        tag: nginx

VM1_app/codebase/backend/observability/docker-compose.yml

@@ -21,6 +21,7 @@ services:
     environment:
       APP_ENV: ${APP_ENV:-production-like}
       RELEASE_VERSION: ${RELEASE_VERSION:-unknown}
+      HOST_NAME: ${HOST_NAME:?HOST_NAME is required}
       OTEL_REMOTE_ENDPOINT: ${OTEL_REMOTE_ENDPOINT}
       OTEL_REMOTE_TLS_INSECURE: ${OTEL_REMOTE_TLS_INSECURE:-false}
     expose: ["4317", "4318", "13133", "8888"]

VM1_app/codebase/backend/observability/otel-collector.yaml

@@ -54,6 +54,7 @@ processors:
       - {key: service.namespace, value: han-chat, action: upsert}
       - {key: deployment.environment, value: "${env:APP_ENV}", action: upsert}
       - {key: service.version, value: "${env:RELEASE_VERSION}", action: upsert}
+      - {key: host.name, value: "${env:HOST_NAME}", action: upsert}
   attributes/redact:
     actions:
       - {key: http.request.header.authorization, action: delete}

VM1_app/codebase/backend/redis/docker-compose.yml

@@ -1,6 +1,8 @@
 services:
   redis:
     image: ${REDIS_IMAGE:?REDIS_IMAGE must be pinned by digest}
+    labels:
+      com.han.service.name: redis
     environment:
       REDIS_MAXMEMORY: ${REDIS_MAXMEMORY:-384mb}
       REDIS_EVICTION_POLICY: ${REDIS_EVICTION_POLICY:-volatile-lru}
@@ -32,7 +34,10 @@ services:
       nofile: {soft: 65536, hard: 65536}
     logging:
       driver: json-file
-      options: {max-size: "50m", max-file: "5"}
+      options:
+        max-size: "50m"
+        max-file: "5"
+        tag: redis
 
 secrets:
   redis_api_password:

VM1_app/codebase/backend/scripts/validate-env

@@ -13,12 +13,13 @@ from urllib.parse import urlparse
 REQUIRED_CONFIG = {
     "SECRETS_SOURCE", "APP_ENV", "RELEASE_VERSION", "HAN_PG_HOST", "HAN_PG_PORT",
     "HAN_PG_DATABASE", "PG_CA_HOST_PATH", "KEYCLOAK_DB_URL", "KEYCLOAK_DB_USERNAME",
-    "PUBLIC_HOST", "PUBLIC_WEB_URL",
+    "PUBLIC_HOST", "PUBLIC_WEB_URL", "HOST_NAME",
     "PUBLIC_API_URL", "PUBLIC_AUTH_URL", "KEYCLOAK_PUBLIC_URL",
     "KEYCLOAK_INTERNAL_URL", "KEYCLOAK_REALM", "KEYCLOAK_SMS_SERVICE_URL",
     "IDGTL_SMS_BASE_URL", "IDGTL_SMS_CALLBACK_PUBLIC_URL",
     "MESSAGE_SAFETY_URL", "MESSAGE_SAFETY_EXTRA_HOST",
     "MESSAGE_SAFETY_CA_HOST_PATH", "MESSAGE_SAFETY_API_PREFIX",
+    "OTEL_REMOTE_ENDPOINT", "OTEL_REMOTE_TLS_INSECURE", "TELEMETRYGEN_IMAGE",
 }
 REQUIRED_RUNTIME = {
     "DATABASE_URL", "BITRIX_DATABASE_URL",

VM1_app/codebase/backend/sms-service/app/logging_security.py

@@ -1,7 +1,7 @@
 from __future__ import annotations
 
 import re
-from collections.abc import Mapping
+from collections.abc import Mapping, MutableMapping
 from typing import Any
 
 REDACTED = "[REDACTED]"
@@ -42,6 +42,6 @@ def sanitize_value(value: Any) -> Any:
 def redact_event(
     _logger: Any,
     _method_name: str,
-    event_dict: dict[str, Any],
+    event_dict: MutableMapping[str, Any],
-) -> dict[str, Any]:
+) -> MutableMapping[str, Any]:
     return sanitize_value(event_dict)

VM1_app/codebase/backend/sms-service/app/main.py

@@ -32,13 +32,18 @@ from app.service import (
     read_message,
 )
 from app.settings import get_settings
-from app.telemetry import add_trace_context, init_telemetry, instrument_fastapi
+from app.telemetry import TelemetryRuntime, add_trace_context, init_telemetry, instrument_fastapi
 
 log = structlog.get_logger()
 
 
-def configure_logging(level: str) -> None:
+def configure_logging(level: str, telemetry: TelemetryRuntime | None = None) -> None:
     logging.basicConfig(level=level, format="%(message)s")
+    if telemetry and telemetry.logging_handler not in logging.getLogger().handlers:
+        telemetry.logging_handler.addFilter(
+            lambda record: not record.name.startswith("opentelemetry")
+        )
+        logging.getLogger().addHandler(telemetry.logging_handler)
     structlog.configure(
         processors=[
             structlog.contextvars.merge_contextvars,
@@ -47,7 +52,10 @@ def configure_logging(level: str) -> None:
             structlog.processors.TimeStamper(fmt="iso", utc=True, key="timestamp"),
             structlog.stdlib.add_log_level,
             structlog.processors.JSONRenderer(),
-        ]
+        ],
+        logger_factory=structlog.stdlib.LoggerFactory(),
+        wrapper_class=structlog.stdlib.BoundLogger,
+        cache_logger_on_first_use=True,
     )
 
 
@@ -55,7 +63,7 @@ def configure_logging(level: str) -> None:
 async def lifespan(app: FastAPI):
     settings = get_settings()
     telemetry = init_telemetry()
-    configure_logging(settings.log_level)
+    configure_logging(settings.log_level, telemetry)
     app.state.settings = settings
     app.state.db = Database(settings.database_url)
     try:

VM1_app/codebase/backend/sms-service/app/telemetry.py

@@ -1,17 +1,23 @@
 from __future__ import annotations
 
+import logging
 import os
+from collections.abc import MutableMapping
 from dataclasses import dataclass
 from typing import Any
 
 from fastapi import FastAPI
 from opentelemetry import metrics, trace
+from opentelemetry._logs import set_logger_provider
+from opentelemetry.exporter.otlp.proto.grpc._log_exporter import OTLPLogExporter
 from opentelemetry.exporter.otlp.proto.grpc.metric_exporter import OTLPMetricExporter
 from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter
 from opentelemetry.instrumentation.fastapi import FastAPIInstrumentor
 from opentelemetry.instrumentation.httpx import HTTPXClientInstrumentor
 from opentelemetry.instrumentation.sqlalchemy import SQLAlchemyInstrumentor
 from opentelemetry.propagate import set_global_textmap
+from opentelemetry.sdk._logs import LoggerProvider, LoggingHandler
+from opentelemetry.sdk._logs.export import BatchLogRecordProcessor
 from opentelemetry.sdk.metrics import MeterProvider
 from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader
 from opentelemetry.sdk.resources import Resource
@@ -25,8 +31,11 @@ from opentelemetry.trace.propagation.tracecontext import TraceContextTextMapProp
 class TelemetryRuntime:
     tracer_provider: TracerProvider
     meter_provider: MeterProvider
+    logger_provider: LoggerProvider
+    logging_handler: LoggingHandler
 
     def shutdown(self) -> None:
+        self.logger_provider.shutdown()
         self.meter_provider.shutdown()
         self.tracer_provider.shutdown()
 
@@ -53,7 +62,9 @@ def init_telemetry(service_name: str | None = None) -> TelemetryRuntime | None:
     if not endpoint:
         return None
 
-    resource = _resource(service_name or os.getenv("OTEL_SERVICE_NAME", "sms-service"))
+    resource = _resource(
+        service_name or os.getenv("OTEL_SERVICE_NAME", "sms-service") or "sms-service"
+    )
     insecure = endpoint.startswith("http://")
     set_global_textmap(TraceContextTextMapPropagator())
 
@@ -77,9 +88,27 @@ def init_telemetry(service_name: str | None = None) -> TelemetryRuntime | None:
     meter_provider = MeterProvider(resource=resource, metric_readers=[metric_reader])
     metrics.set_meter_provider(meter_provider)
 
+    logger_provider = LoggerProvider(resource=resource)
+    logger_provider.add_log_record_processor(
+        BatchLogRecordProcessor(
+            OTLPLogExporter(endpoint=endpoint, insecure=insecure, timeout=3),
+            max_queue_size=2048,
+            schedule_delay_millis=5000,
+            max_export_batch_size=512,
+            export_timeout_millis=3000,
+        )
+    )
+    set_logger_provider(logger_provider)
+    logging_handler = LoggingHandler(level=logging.NOTSET, logger_provider=logger_provider)
+
     HTTPXClientInstrumentor().instrument()
     SQLAlchemyInstrumentor().instrument(enable_commenter=False)
-    _runtime = TelemetryRuntime(tracer_provider, meter_provider)
+    _runtime = TelemetryRuntime(
+        tracer_provider,
+        meter_provider,
+        logger_provider,
+        logging_handler,
+    )
     return _runtime
 
 
@@ -93,8 +122,8 @@ def instrument_fastapi(app: FastAPI) -> None:
 def add_trace_context(
     _logger: Any,
     _method_name: str,
-    event_dict: dict[str, Any],
+    event_dict: MutableMapping[str, Any],
-) -> dict[str, Any]:
+) -> MutableMapping[str, Any]:
     context = trace.get_current_span().get_span_context()
     if context.is_valid:
         event_dict["trace_id"] = format(context.trace_id, "032x")

VM1_app/codebase/backend/sms-service/app/worker.py

@@ -10,6 +10,7 @@ from datetime import UTC, datetime, timedelta
 import httpx
 import structlog
 from opentelemetry import trace
+from opentelemetry.propagate import extract
 from prometheus_client import start_http_server
 from sqlalchemy import and_, func, or_, select, update
 
@@ -26,14 +27,27 @@ from app.metrics import (
 from app.provider import IdgtlClient, IdgtlConfig
 from app.service import RuntimeSettings, load_runtime_settings
 from app.settings import Settings, get_settings
-from app.telemetry import add_trace_context, init_telemetry
+from app.telemetry import TelemetryRuntime, add_trace_context, init_telemetry
 
 log = structlog.get_logger()
 MAX_CONNECT_ATTEMPTS = 3
 
 
-def configure_logging(level: str) -> None:
+def origin_links(traceparent: str | None) -> list[trace.Link]:
+    if not traceparent:
+        return []
+    context = extract({"traceparent": traceparent})
+    span_context = trace.get_current_span(context).get_span_context()
+    return [trace.Link(span_context)] if span_context.is_valid else []
+
+
+def configure_logging(level: str, telemetry: TelemetryRuntime | None = None) -> None:
     logging.basicConfig(level=level, format="%(message)s")
+    if telemetry and telemetry.logging_handler not in logging.getLogger().handlers:
+        telemetry.logging_handler.addFilter(
+            lambda record: not record.name.startswith("opentelemetry")
+        )
+        logging.getLogger().addHandler(telemetry.logging_handler)
     structlog.configure(
         processors=[
             structlog.contextvars.merge_contextvars,
@@ -42,7 +56,10 @@ def configure_logging(level: str) -> None:
             structlog.processors.TimeStamper(fmt="iso", utc=True, key="timestamp"),
             structlog.stdlib.add_log_level,
             structlog.processors.JSONRenderer(),
-        ]
+        ],
+        logger_factory=structlog.stdlib.LoggerFactory(),
+        wrapper_class=structlog.stdlib.BoundLogger,
+        cache_logger_on_first_use=True,
     )
 
 
@@ -177,7 +194,7 @@ async def update_queue_metrics(db: Database) -> None:
 async def worker_loop(stop: asyncio.Event) -> None:
     settings = get_settings()
     telemetry = init_telemetry("sms-worker")
-    configure_logging(settings.log_level)
+    configure_logging(settings.log_level, telemetry)
     structlog.contextvars.bind_contextvars(**{"service.name": "sms-worker"})
     metrics_server, metrics_thread = start_http_server(
         settings.metrics_port,
@@ -200,8 +217,17 @@ async def worker_loop(stop: asyncio.Event) -> None:
                         await update_queue_metrics(db)
                         await asyncio.wait_for(stop.wait(), timeout=runtime.poll_interval_ms / 1000)
                         continue
-                    process_span = tracer.start_span("sms.process", start_time=claim_started_ns)
+                    process_span = tracer.start_span(
-                    with trace.use_span(process_span, end_on_exit=True):
+                        "sms.process",
+                        start_time=claim_started_ns,
+                        links=origin_links(message.traceparent),
+                    )
+                    with (
+                        trace.use_span(process_span, end_on_exit=True),
+                        structlog.contextvars.bound_contextvars(
+                            sms_message_id=str(message.id)
+                        ),
+                    ):
                         claim_span = tracer.start_span("sms.claim", start_time=claim_started_ns)
                         claim_span.set_attribute("sms.claimed", True)
                         claim_span.end(end_time=claim_finished_ns)

VM1_app/codebase/backend/sms-service/tests/unit/test_telemetry.py

@@ -1,6 +1,7 @@
 from opentelemetry.sdk.trace import TracerProvider
 
 from app import telemetry
+from app.worker import origin_links
 
 
 def test_telemetry_is_fail_open_without_endpoint(monkeypatch) -> None:
@@ -19,3 +20,8 @@ def test_structlog_processor_adds_active_trace_context() -> None:
     assert result["event"] == "sms.safe"
     assert len(result["trace_id"]) == 32
     assert len(result["span_id"]) == 16
+
+
+def test_sms_worker_rejects_invalid_origin_context() -> None:
+    assert origin_links(None) == []
+    assert origin_links("00-invalid") == []

VM1_app/codebase/backend/tests/test_config.py

@@ -348,9 +348,63 @@ class InfrastructureConfigTests(unittest.TestCase):
             source = service_main.read_text(encoding="utf-8")
             self.assertNotIn("route=request.url.path", source)
             self.assertIn('getattr(request.scope.get("route"), "path"', source)
+        for telemetry_path in (
+            ROOT / "api-backend/app/telemetry.py",
+            ROOT / "sms-service/app/telemetry.py",
+            ROOT / "bitrix-local-app/app/telemetry.py",
+        ):
+            telemetry = telemetry_path.read_text(encoding="utf-8")
+            self.assertIn("OTLPLogExporter", telemetry)
+            self.assertIn("BatchLogRecordProcessor", telemetry)
+            self.assertIn("LoggingHandler", telemetry)
         worker = (ROOT / "sms-service/app/worker.py").read_text(encoding="utf-8")
         for span_name in ("sms.claim", "sms.process", "sms.provider", "sms.save_result"):
             self.assertIn(f'"{span_name}"', worker)
+        bitrix = (ROOT / "bitrix-local-app/app/main.py").read_text(encoding="utf-8")
+        self.assertIn('EXPECTED_BITRIX_DB_REVISION = "0002_inbox_trace_context"', bitrix)
+        self.assertIn("links=origin_links(row.traceparent)", bitrix)
+        bitrix_migration = (
+            ROOT / "bitrix-local-app/alembic/versions/0002_inbox_trace_context.py"
+        ).read_text(encoding="utf-8")
+        self.assertIn('down_revision: str | None = "0001_bitrix_local"', bitrix_migration)
+
+    def test_host_otel_collector_is_hardened_and_allowlisted(self) -> None:
+        config = (
+            ROOT / "deployment/observability/otel-host-collector.yaml"
+        ).read_text(encoding="utf-8")
+        unit = (ROOT / "deployment/han-host-otel-collector@.service").read_text(
+            encoding="utf-8"
+        )
+        setup = (ROOT / "deployment/scripts/setup-vm.sh").read_text(encoding="utf-8")
+        for required in (
+            "filelog/docker",
+            "journald/host",
+            "filter/allowlist",
+            "storage: file_storage",
+            'layout: "2006-01-02T15:04:05.000000000Z07:00"',
+            'set(body, "nginx.access")',
+            'set(body, "keycloak.event")',
+            '{key: uri, action: delete}',
+            'resource.attributes["service.name"] != "nginx"',
+            'resource.attributes["service.name"] != "keycloak"',
+            'resource.attributes["service.name"] != "redis"',
+        ):
+            self.assertIn(required, config)
+        self.assertNotIn("/var/run/docker.sock", config + unit)
+        self.assertNotIn('from: attributes["service.name"]', config)
+        self.assertIn("CapabilityBoundingSet=", unit)
+        self.assertIn("NoNewPrivileges=yes", unit)
+        self.assertIn("ReadOnlyPaths=/var/lib/docker/containers", unit)
+        self.assertIn("ReadWritePaths=/var/lib/han-otel/host-collector", unit)
+        self.assertIn("OTEL_HOST_COLLECTOR_SHA256", setup)
+        self.assertIn("sha256sum -c -", setup)
+        self.assertIn("journalctl --flush", setup)
+        self.assertNotIn("systemctl start han-host-otel-collector", setup)
+        verify = (
+            ROOT / "deployment/scripts/verify-observability.sh"
+        ).read_text(encoding="utf-8")
+        self.assertIn("TELEMETRYGEN_IMAGE", verify)
+        self.assertNotIn("telemetrygen:latest", verify)
 
     def test_settings_cli_and_workers_are_deployable(self) -> None:
         cli = ROOT / "api-backend/app/cli"
@@ -442,7 +496,7 @@ class InfrastructureConfigTests(unittest.TestCase):
             dedup_fix,
         )
         self.assertIn('down_revision: str | None = "0009_chat_message_max"', dedup_fix)
-        self.assertIn('EXPECTED_API_DB_REVISION = "0012_safety_v2_checkpoint"', main)
+        self.assertIn('EXPECTED_API_DB_REVISION = "0013_delivery_trace_context"', main)
 
     def test_consent_audit_migration_supports_existing_and_fresh_databases(self) -> None:
         migration = (

VM1_app/documentation/module-09-observability-vm1.md

@@ -22,8 +22,14 @@
 | Keycloak | `keycloak` |
 | `sms-service` | `sms-service` |
 | `sms-worker` | `sms-worker` |
+| delivery worker | `delivery-worker` |
+| safety recovery worker | `safety-recovery-worker` |
+| quarantine cleanup worker | `cleanup-worker` |
+| notification expiration worker | `notification-expire-worker` |
+| notification draft cleanup worker | `notification-draft-cleanup-worker` |
 | Redis DB0/DB1 | `redis` |
 | local Collector | `otel-collector` |
+| host telemetry agent | `otel-host-collector` |
 
 SMS-метрики и алерты детализированы в [`module-11-idgtl-sms.md`](module-11-idgtl-sms.md); имена зарегистрированы в arch-07 §4.
 
@@ -36,6 +42,22 @@ SMS-метрики и алерты детализированы в [`module-11-i
 - export в SigNoz `192.168.0.5:4317`; hostname collector ВМ2 не используется;
 - pipeline, processors, limits, `otel-queue-init` и fail-open — arch-07 §3, §13, §14.
 
+Python-сервисы экспортируют structured logs через OTLP Logs SDK в Compose
+Collector. JSON stdout сохраняется только как bounded аварийный buffer и не
+собирается повторно.
+
+Отдельный hardened host agent собирает по allow-list:
+
+- Docker `json-file` для nginx, Keycloak и Redis;
+- journal units `docker`, `han-stack`, `han-secrets`, Docker firewall,
+  certbot и fail2ban;
+- UFW log, если он включён на host.
+
+Host agent не имеет доступа к Docker socket, исключает собственные логи и
+Compose Collector, хранит offsets/queue в `/var/lib/han-otel/host-collector`
+и экспортирует напрямую в тот же SigNoz. Platform events без активного span
+не получают искусственные trace IDs и ищутся по host/service/time window.
+
 Scrape targets ВМ1 (кроме самого Collector): Keycloak management metrics, Redis exporter приложения, nginx exporter, сервисные `/metrics` `api-backend` и `bitrix-local-app`, если они не идут OTLP.
 
 ## 4. Instrumentation

VM4_Expo-mobile/app.config.ts

@@ -13,6 +13,9 @@ const config: ExpoConfig = {
     package: "ru.han.chat",
     versionCode: 2,
     softwareKeyboardLayoutMode: "resize",
+    // Expo prebuild template injects this into every AAB. The app does not draw
+    // over other windows; RuStore/Play treat it as a sensitive overlay permission.
+    blockedPermissions: ["android.permission.SYSTEM_ALERT_WINDOW"],
     adaptiveIcon: {
       foregroundImage: "./.assets/icons/adaptive-foreground.png",
       monochromeImage: "./.assets/icons/adaptive-monochrome.png",

architectory/arch-07-observability.md

@@ -184,12 +184,35 @@ service:
 - `deployment.environment=production-like|production`;
 - `host.name`/`service.instance.id` без публичного IP.
 
+Для logs действуют те же resource attributes, что для traces/metrics.
+`trace_id` (32 hex) и `span_id` (16 hex) добавляются только из активного
+OpenTelemetry span. SigNoz связывает log со span по этой паре. Host, Redis,
+Docker и systemd события, у которых span отсутствует, коррелируются по
+`host.name`, `service.name`, времени и безопасным request/business IDs;
+синтетические trace/span IDs для них запрещены.
+
+На application VM используются два непересекающихся канала:
+
+- Python application logs отправляются OTLP Logs SDK в Compose Collector и
+  одновременно остаются в JSON stdout как локальный аварийный buffer;
+- platform logs (nginx, Keycloak, Redis, Docker и выбранные systemd units)
+  собирает отдельный host agent без доступа к Docker socket.
+
+Один источник не может одновременно экспортироваться SDK и host `filelog`.
+Allow-list источников и self-exclusion Collector обязательны, чтобы исключить
+дубли и feedback loop.
+
 Реестр `service.name`:
 
 | Имя | Владелец |
 |---|---|
 | `nginx` | nginx каждой VM; различать экземпляры `host.name` / `service.instance.id` |
 | `api-backend` | ВМ1 |
+| `delivery-worker` | ВМ1 |
+| `safety-recovery-worker` | ВМ1 |
+| `cleanup-worker` | ВМ1 |
+| `notification-expire-worker` | ВМ1 |
+| `notification-draft-cleanup-worker` | ВМ1 |
 | `bitrix-local-app` | ВМ1 |
 | `keycloak` | ВМ1 |
 | `sms-service` | ВМ1, callback/internal API |
@@ -198,6 +221,7 @@ service:
 | `bitrix-sync` | ВМ2 |
 | `redis` | Redis каждой VM; различать экземпляры |
 | `otel-collector` | collector каждой VM |
+| `otel-host-collector` | host telemetry agent ВМ1 |
 
 Новое имя добавляется только сюда, затем в спецификацию владельца.
 

support&notes/deploy_stories/260903_2.md

@@ -1,5 +1,9 @@
 Поддержка версионирования приложений
 
+# update images
+API_BACKEND_IMAGE=cr.selcloud.ru/han-images/han-api-backend@sha256:ee1e1b94eb944c44eb60534aeb003f36636b1e9a3df4fbf82d5b1a3a955a5e34
+NGINX_IMAGE=cr.selcloud.ru/han-images/nginx@sha256:c64e5a6296ca183538306fef9ae49a3168637bd8a4c925884016da341691673f
+
 # Копируем файл на ВМ1 (HAN_chat_specification\VM1_app\codebase\backend\deployment\app-settings.production-like.yaml)
 
 cd C:/Users/MI/Documents/Assistent/HAN_chat_specification/VM1_app/
@@ -39,3 +43,6 @@ editor /etc/han/vm1.env
 /usr/local/sbin/han-vm1-compose ps --format "table {{.Service}}\t{{.Status}}\t{{.Ports}}"
 
 # Сборка тестовой версии приложения
+cd C:\Users\MI\Documents\Assistent\HAN_chat_specification\VM4_Expo-mobile
+npx eas-cli build --profile preview-rustore --platform android
+npx eas-cli build --profile rustore --platform android

support&notes/deploy_stories/260904_1.md

@@ -0,0 +1,127 @@
+Логирование операций
+
+1. env:
+
+изменение Release_Version
+ + две переменные:
+HOST_NAME=devhanapp \ prodhanapp.ihan.ru.
+TELEMETRYGEN_IMAGE=ghcr.io/open-telemetry/opentelemetry-collector-contrib/telemetrygen@sha256:9461e2c7213219467e5f775ea8652d15c990afb4d1ca94e62344f2025a7587d1
+
+2. Обновить образы:
+
+API_BACKEND_IMAGE=cr.selcloud.ru/han-images/han-api-backend@sha256:5ec8336ce56de41880398f68a2174530ba4dde9cd8ae6151c42f1631ab29bc35
+SMS_SERVICE_IMAGE=cr.selcloud.ru/han-images/sms-service@sha256:404685c45b7391a78f95c014ce8fdff64a84ccd7ae5b280e7a65b524c109d898
+BITRIX_LOCAL_APP_IMAGE=cr.selcloud.ru/han-images/bitrix-local-app@sha256:442889dc20e76f2a16d093adb4ae3ae38508535bafda82654755d1da50693183
+NGINX_IMAGE=cr.selcloud.ru/han-images/nginx@sha256:2cd7d32967b400c15bb6aaa18e97e99d429b2b6f2bfd54b965fc7d42a91a228e
+
+3. Скопировать проект:
+
+```powershell
+$Release = "1.0.1"
+tar --exclude=backend/.env `
+  --exclude='backend/**/__pycache__' `
+  --exclude='backend/**/.pytest_cache' `
+  --exclude='backend/**/.ruff_cache' `
+  -czf "vm1-backend-$Release.tar.gz" `
+  -C C:/Users/MI/Documents/Assistent/HAN_chat_specification/VM1_app/codebase backend
+Get-FileHash "vm1-backend-$Release.tar.gz" -Algorithm SHA256
+scp "vm1-backend-$Release.tar.gz" devVM1Deploy:/var/lib/han-deploy/incoming/
+```
+
+RELEASE='1.0.1'
+EXPECTED_SHA256='900C7AF2AEC061CFA112F82F8511A8A21F43EDB356EA3EF8C3205F49B6F1DF8F'
+ARCHIVE="/var/lib/han-deploy/incoming/vm1-backend-${RELEASE}.tar.gz"
+printf '%s  %s/n' "$EXPECTED_SHA256" "$ARCHIVE" | sha256sum --check -
+tar -tvzf "$ARCHIVE"
+if tar -tzf "$ARCHIVE" | grep -Eq '(^/|(^|/)\.\.(/|$)|^backend/\.env$)'; then
+  echo 'unsafe path or .env' >&2; exit 1
+fi
+if tar -tzf "$ARCHIVE" | grep -Ev '^backend(/|$)' | grep -q .; then
+  echo 'archive has files outside backend' >&2; exit 1
+fi
+if tar -tvzf "$ARCHIVE" | awk '$1 ~ /^[lh]/ {found=1} END {exit !found}'; then
+  echo 'symlink/hardlink is forbidden' >&2; exit 1
+fi
+
+TARGET="/opt/han-chat/releases/${RELEASE}"
+test ! -e "$TARGET"
+install -d -m 0755 -o root -g root "$TARGET"
+tar --extract --gzip --file "$ARCHIVE" --directory "$TARGET" \
+  --no-same-owner --no-same-permissions
+test -f "$TARGET/backend/docker-compose.yml"
+chmod 0755 \
+  "$TARGET/backend/deployment/scripts/setup-vm.sh" \
+  "$TARGET/backend/deployment/preflight.sh" \
+  "$TARGET/backend/deployment/scripts/tls-deploy-hook.sh" \
+  "$TARGET/backend/deployment/secrets/han-compose" \
+  "$TARGET/backend/deployment/secrets/han-secrets"
+test -x "$TARGET/backend/deployment/scripts/setup-vm.sh"
+test -x "$TARGET/backend/deployment/preflight.sh"
+test -x "$TARGET/backend/deployment/scripts/tls-deploy-hook.sh"
+test -x "$TARGET/backend/deployment/secrets/han-compose"
+test -x "$TARGET/backend/deployment/secrets/han-secrets"
+if find "$TARGET/backend" -type l -print -quit | grep -q .; then exit 1; fi
+chown -R root:root "$TARGET"
+chmod -R go-w "$TARGET"
+ln -s "releases/${RELEASE}" /opt/han-chat/.current-new
+mv -Tf /opt/han-chat/.current-new /opt/han-chat/current
+
+cd /opt/han-chat/current/backend/
+sudo apt update
+sudo apt install -y dos2unix
+find . -type f \( \
+  -name '*.sh' -o -name '*.py' -o -name '*.service' -o -name '*.sudoers' -o -name 'han-compose' -o -name 'han-secrets' -o -name 'han-message-safety-mode' -o -name '*.yml' -o -name '*.conf*' -o -name 'preflight.sh' -o -name '*.yaml' -o -name '*.md' -o -name '*.toml' -o -name 'Dockerfile' -o -name '*.acl*' -o -name '*.tsx' -o -name '*.ts' -o -name '*.java' -o -name '*.js' -o -name '*.ps1' -o -name '*.xml' -o -name '*.dockerignore' -o -name '*.env*' -o -name '*.css' -o -name '*.ftl' -o -name '*.TAG' -o -name '*validate-env*' -o -name '*.properties' -o -name '*org.keycloak.*Factory' -o -name '*.html' \
+\) -exec dos2unix {} +
+find -type f -exec file {} \; | grep -i 'CRLF'
+
+
+sudo apt remove -y dos2unix
+sudo apt purge -y dos2unix
+
+
+/usr/local/sbin/han-vm1-compose --profile ops run --rm migrate-api alembic current
+/usr/local/sbin/han-vm1-compose --profile ops run --rm migrate-bitrix-local alembic current
+
+/usr/local/sbin/han-vm1-compose --profile ops run --rm migrate-api
+/usr/local/sbin/han-vm1-compose --profile ops run --rm migrate-bitrix-local
+
+Проверка применения миграции
+/usr/local/sbin/han-vm1-compose --profile ops run --rm migrate-bitrix-local alembic current
+/usr/local/sbin/han-vm1-compose --profile ops run --rm migrate-bitrix-local alembic heads
+
+
+scp `
+    C:/Users/MI/.ssh/devDeploy.pub `
+    C:/Users/MI/.ssh/devAdmin.pub `
+devVM1Deploy:/var/lib/han-deploy/incoming/
+
+install -m 0600 -o root -g root /var/lib/han-deploy/incoming/devDeploy.pub /root/bootstrap/deploy.pub
+install -m 0600 -o root -g root /var/lib/han-deploy/incoming/devAdmin.pub /root/bootstrap/admin.pub
+
+rm -f /var/lib/han-deploy/incoming/devDeploy.pub /var/lib/han-deploy/incoming/devAdmin.pub
+
+cd /opt/han-chat/current/backend
+
+export OTEL_HOST_COLLECTOR_VERSION='0.117.0'
+export OTEL_HOST_COLLECTOR_SHA256='90710c909a30fc3b89dd0e389c13f9e57ebbb87d6a0dc51fdde0bf03499a5f25'
+DEPLOY_AUTHORIZED_KEY_FILE=/root/bootstrap/deploy.pub \
+ADMIN_AUTHORIZED_KEY_FILE=/root/bootstrap/admin.pub \
+HARDEN_SSH=true SKIP_APT_UPGRADE=true \
+/opt/han-chat/current/backend/deployment/scripts/setup-vm.sh
+
+/usr/local/bin/otelcol-contrib validate \
+  --config=/opt/han-chat/current/backend/deployment/observability/otel-host-collector.yaml
+
+chmod 0755 /opt/han-chat/current/backend/scripts/validate-env
+/opt/han-chat/current/backend/deployment/preflight.sh
+
+systemctl restart han-stack@production.service
+если ошибка systemctl status han-stack@production.service
+docker pull cr.selcloud.ru/han-images/nginx@sha256:2cd7d32967b400c15bb6aaa18e97e99d429b2b6f2bfd54b965fc7d42a91a228e
+docker pull cr.selcloud.ru/han-images/sms-service@sha256:404685c45b7391a78f95c014ce8fdff64a84ccd7ae5b280e7a65b524c109d898
+
+systemctl start han-host-otel-collector@production.service
+
+Canary test: CONFIG_FILE=/etc/han/vm1.env ./deployment/scripts/verify-observability.sh
+
+rm -f /root/bootstrap/deploy.pub /root/bootstrap/admin.pub