Admin/han-app
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Внедрение KESL на ВМ2 + замена CLAMAV на KESL

Browse Source
This commit is contained in:
mi
2026-09-08 01:39:37 +03:00
parent 85df788f2d
commit fdfdeaffb4
43 changed files with 2210 additions and 329 deletions
Download Patch File Download Diff File Expand all files Collapse all files
VM2_services/codebase/services/message-safety/app/api.py
+1
View File
@@ -96,6 +96,7 @@ def create_app(service: SafetyService, token: str) -> FastAPI:
@app.get("/health/ready")
async def ready() -> JSONResponse:
await service.refresh_antivirus()
mode = "mock" if service.mode.mock else "standard"
components = {
"postgres": "ok",
VM2_services/codebase/services/message-safety/app/artifacts/config.schema.json
+9 -5
View File
@@ -2,9 +2,9 @@
"$schema": "https://json-schema.org/draft/2020-12/schema",
"type": "object",
"additionalProperties": false,
"required": ["schema_version", "rules_bundle_ref", "detector_manifest_ref", "task", "rate", "retention", "cache", "link", "clamav", "file_policy"],
"required": ["schema_version", "rules_bundle_ref", "detector_manifest_ref", "task", "rate", "retention", "cache", "link", "antivirus", "file_policy"],
"properties": {
"schema_version": {"const": 1},
"schema_version": {"const": 2},
"rules_bundle_ref": {"type": "string", "pattern": "^rules-[0-9]{4}-[0-9]{2}-[0-9]{2}$"},
"detector_manifest_ref": {"const": "detector-2026-08-03"},
"task": {
@@ -48,9 +48,13 @@
"pipeline_timeout_sec": {"type": "number", "exclusiveMinimum": 0, "maximum": 5}
}
},
"clamav": {
"type": "object", "additionalProperties": false, "required": ["scan_timeout_sec", "max_signature_age_hours"],
"properties": {"scan_timeout_sec": {"type": "integer", "minimum": 1, "maximum": 120}, "max_signature_age_hours": {"type": "integer", "minimum": 1, "maximum": 720}}
"antivirus": {
"type": "object", "additionalProperties": false, "required": ["engine", "scan_timeout_sec", "max_signature_age_hours"],
"properties": {
"engine": {"const": "kesl"},
"scan_timeout_sec": {"type": "integer", "minimum": 1, "maximum": 300},
"max_signature_age_hours": {"type": "integer", "minimum": 1, "maximum": 720}
}
},
"file_policy": {
"type": "object", "additionalProperties": false, "required": ["enabled_mime_types", "max_size_bytes"],
VM2_services/codebase/services/message-safety/app/artifacts/seed-config.yaml
+4 -3
View File
@@ -1,4 +1,4 @@
schema_version: 1
schema_version: 2
rules_bundle_ref: rules-2026-01-01
detector_manifest_ref: detector-2026-08-03
task:
@@ -21,8 +21,9 @@ link:
url_max_length: 2048
dns_lookup_timeout_sec: 1
pipeline_timeout_sec: 2
clamav:
scan_timeout_sec: 45
antivirus:
engine: kesl
scan_timeout_sec: 60
max_signature_age_hours: 240
file_policy:
enabled_mime_types:
VM2_services/codebase/services/message-safety/app/file_pipeline.py
+110 -55
View File
@@ -7,6 +7,8 @@ import json
import struct
from collections.abc import AsyncIterator
from dataclasses import dataclass
from datetime import UTC, datetime
from email.utils import parsedate_to_datetime
from pathlib import Path
from typing import Protocol
@@ -33,7 +35,24 @@ class ObjectReader(Protocol):
class Antivirus(Protocol):
async def scan(self, chunks: AsyncIterator[bytes]) -> str | None: ...
async def scan(
self, chunks: AsyncIterator[bytes], *, scan_timeout: float
) -> AntivirusScanResult: ...
async def status(self) -> AntivirusStatus: ...
@dataclass(frozen=True)
class AntivirusScanResult:
threat: str | None
signatures_version: str
@dataclass(frozen=True)
class AntivirusStatus:
engine_version: str
signatures_version: str
databases_date: datetime
@dataclass(frozen=True)
@@ -119,65 +138,101 @@ async def collect_and_hash(
return bytes(body), digest.digest()
class ClamAvInstream:
def __init__(self, host: str, port: int, timeout: float = 45.0) -> None:
self.host, self.port, self.timeout = host, port, timeout
class KeslSocketScanner:
MAX_HEADER = 4096
MAX_RESPONSE = 16 * 1024
async def scan(self, chunks: AsyncIterator[bytes]) -> str | None:
def __init__(self, socket_path: Path, timeout: float = 60.0) -> None:
self.socket_path = socket_path
self.timeout = timeout
async def _request(
self,
header: dict[str, object],
body: bytes = b"",
*,
request_timeout: float | None = None,
) -> dict[str, object]:
encoded = json.dumps(header, separators=(",", ":")).encode()
if len(encoded) > self.MAX_HEADER:
raise DependencyFailure("KESL request header is too large")
async def operation() -> dict[str, object]:
reader, writer = await asyncio.open_unix_connection(str(self.socket_path))
try:
writer.write(struct.pack(">I", len(encoded)) + encoded + body)
await writer.drain()
(size,) = struct.unpack(">I", await reader.readexactly(4))
if size < 2 or size > self.MAX_RESPONSE:
raise DependencyFailure("invalid KESL broker response size")
value = json.loads((await reader.readexactly(size)).decode())
if not isinstance(value, dict):
raise DependencyFailure("invalid KESL broker response")
return value
finally:
writer.close()
await writer.wait_closed()
try:
return await asyncio.wait_for(operation(), request_timeout or self.timeout)
except (OSError, TimeoutError, asyncio.IncompleteReadError, json.JSONDecodeError) as exc:
raise DependencyFailure("KESL unavailable") from exc
async def scan(
self, chunks: AsyncIterator[bytes], *, scan_timeout: float
) -> AntivirusScanResult:
body = bytearray()
async for chunk in chunks:
body.extend(chunk)
with tracer.start_as_current_span(
"message_safety.clamav.scan",
attributes={"server.address.type": "clamav"},
"message_safety.antivirus.scan",
attributes={"message_safety.antivirus.engine": "kesl"},
):
return await self._scan(chunks)
async def _scan(self, chunks: AsyncIterator[bytes]) -> str | None:
async def operation() -> str | None:
reader, writer = await asyncio.open_connection(self.host, self.port)
try:
writer.write(b"zINSTREAM\0")
async for chunk in chunks:
writer.write(struct.pack(">I", len(chunk)) + chunk)
await writer.drain()
writer.write(struct.pack(">I", 0))
await writer.drain()
result = await reader.readuntil(b"\0")
text = result.rstrip(b"\0").decode("utf-8", "replace")
if text.endswith(" OK"):
return None
if text.endswith(" FOUND"):
return text.rsplit(": ", 1)[-1].removesuffix(" FOUND")
raise DependencyFailure("invalid ClamAV response")
finally:
writer.close()
await writer.wait_closed()
try:
return await asyncio.wait_for(operation(), self.timeout)
except (OSError, TimeoutError) as exc:
raise DependencyFailure("ClamAV unavailable") from exc
async def signatures_version(self) -> str:
with tracer.start_as_current_span("message_safety.clamav.version"):
return await self._signatures_version()
async def _signatures_version(self) -> str:
try:
reader, writer = await asyncio.wait_for(
asyncio.open_connection(self.host, self.port), 2.0
value = await self._request(
{"op": "scan", "size": len(body)},
bytes(body),
request_timeout=scan_timeout,
)
if value.get("status") != "completed" or value.get("verdict") not in {
"clean",
"infected",
}:
raise DependencyFailure("KESL scan failed")
signatures = value.get("signatures_version")
if not isinstance(signatures, str) or not signatures.startswith("sha256:"):
raise DependencyFailure("KESL signatures version is missing")
threat = value.get("threat")
if value["verdict"] == "infected" and not isinstance(threat, str):
raise DependencyFailure("KESL infected verdict has no threat")
return AntivirusScanResult(
threat=threat if isinstance(threat, str) else None,
signatures_version=signatures,
)
async def status(self) -> AntivirusStatus:
with tracer.start_as_current_span(
"message_safety.antivirus.status",
attributes={"message_safety.antivirus.engine": "kesl"},
):
value = await self._request({"op": "status"})
if value.get("status") != "ready":
raise DependencyFailure("KESL is not ready")
try:
database_date = datetime.fromisoformat(
str(value["databases_date"]).replace("Z", "+00:00")
)
except ValueError:
try:
writer.write(b"zVERSION\0")
await writer.drain()
raw = await asyncio.wait_for(reader.readuntil(b"\0"), 2.0)
finally:
writer.close()
await writer.wait_closed()
except (OSError, TimeoutError) as exc:
raise DependencyFailure("ClamAV unavailable") from exc
value = raw.rstrip(b"\0")
if not value.startswith(b"ClamAV ") or len(value) > 512:
raise DependencyFailure("invalid ClamAV version response")
return "sha256:" + hashlib.sha256(value).hexdigest()
database_date = parsedate_to_datetime(str(value["databases_date"]))
except (TypeError, ValueError) as exc:
raise DependencyFailure("invalid KESL database date") from exc
if database_date.tzinfo is None:
database_date = database_date.replace(tzinfo=UTC)
engine = value.get("engine_version")
signatures = value.get("signatures_version")
if not isinstance(engine, str) or not isinstance(signatures, str):
raise DependencyFailure("incomplete KESL status")
return AntivirusStatus(engine, signatures, database_date.astimezone(UTC))
async def one_chunk(data: bytes) -> AsyncIterator[bytes]:
VM2_services/codebase/services/message-safety/app/main.py
+5 -3
View File
@@ -8,7 +8,7 @@ from app.adapters import TrustedDnsResolver
from app.api import create_app
from app.config import ActiveConfig, validate_config
from app.db import engine_and_sessions
from app.file_pipeline import ClamAvInstream, DependencyFailure
from app.file_pipeline import DependencyFailure, KeslSocketScanner
from app.repository import Repository
from app.service import SafetyService
from app.settings import BootstrapSettings, EmergencyMode
@@ -29,13 +29,14 @@ async def build_runtime() -> tuple[object, object]:
resolver = TrustedDnsResolver(
[item.strip() for item in settings.dns_resolvers.split(",") if item.strip()]
)
clamav = ClamAvInstream(settings.clamav_host, settings.clamav_port)
antivirus = KeslSocketScanner(settings.antivirus_socket)
if mode.mock:
signatures_version = "unavailable"
files_ready = False
else:
try:
signatures_version = await clamav.signatures_version()
status = await antivirus.status()
signatures_version = status.signatures_version
files_ready = True
except DependencyFailure:
signatures_version = "unavailable"
@@ -47,6 +48,7 @@ async def build_runtime() -> tuple[object, object]:
resolver,
files_ready=files_ready,
signatures_version=signatures_version,
antivirus=antivirus,
)
app = create_app(service, settings.service_token.get_secret_value())
instrument_fastapi(app)
VM2_services/codebase/services/message-safety/app/repository.py
+23 -13
View File
@@ -90,7 +90,7 @@ class Repository:
FileVerdictCache.config_version == config.version,
FileVerdictCache.rules_version == config.rules_version,
FileVerdictCache.detector_version == config.detector.version,
FileVerdictCache.scanner_engine == "clamav",
FileVerdictCache.scanner_engine == "kesl",
FileVerdictCache.signatures_version == signatures_version,
FileVerdictCache.expires_at > func.now(),
)
@@ -262,9 +262,29 @@ class Repository:
return result.rowcount == 1
async def finish(
self, task_id: uuid.UUID, owner: str, generation: int, *, allow: bool, rule_id: str
self,
task_id: uuid.UUID,
owner: str,
generation: int,
*,
allow: bool,
rule_id: str,
signatures_version: str | None = None,
) -> bool:
now = datetime.now(UTC)
values = {
"status": TaskStatus.allowed if allow else TaskStatus.denied,
"verdict": "allow" if allow else "deny",
"rule_id": rule_id,
"reason_code": None if allow else "message_blocked",
"finished_at": now,
"purge_after": now + timedelta(days=30),
"updated_at": now,
"lease_owner": None,
"lease_until": None,
}
if signatures_version is not None:
values["signatures_version"] = signatures_version
async with self.sessions.begin() as session:
result = await session.execute(
update(SafetyTask)
@@ -275,17 +295,7 @@ class Repository:
SafetyTask.lease_generation == generation,
SafetyTask.lease_until > func.now(),
)
.values(
status=TaskStatus.allowed if allow else TaskStatus.denied,
verdict="allow" if allow else "deny",
rule_id=rule_id,
reason_code=None if allow else "message_blocked",
finished_at=now,
purge_after=now + timedelta(days=30),
updated_at=now,
lease_owner=None,
lease_until=None,
)
.values(**values)
)
if result.rowcount == 1:
await session.execute(
VM2_services/codebase/services/message-safety/app/service.py
+34 -3
View File
@@ -1,5 +1,6 @@
from __future__ import annotations
import asyncio
import uuid
from datetime import UTC, datetime, timedelta
from time import perf_counter
@@ -14,7 +15,7 @@ from app.db import (
TaskStatus,
TextRulesCache,
)
from app.file_pipeline import validate_metadata
from app.file_pipeline import Antivirus, DependencyFailure, validate_metadata
from app.fingerprint import fingerprint
from app.normalization import normalize_text
from app.rate_limit import ConservativeRateLimiter
@@ -52,6 +53,7 @@ class SafetyService:
links_ready: bool = True,
files_ready: bool = True,
signatures_version: str = "unverified",
antivirus: Antivirus | None = None,
) -> None:
self.repository = repository
self.config = config
@@ -60,11 +62,40 @@ class SafetyService:
self.links_ready = links_ready
self.files_ready = files_ready
self.signatures_version = signatures_version
self.antivirus = antivirus
self._antivirus_checked_at = 0.0
self._antivirus_check_lock = asyncio.Lock()
self.rate_limiter = ConservativeRateLimiter(
config.document["rate"]["text_rps"], config.document["rate"]["file_rps"]
)
record_runtime_state("mock" if mode.mock else "standard", config.version)
async def refresh_antivirus(self) -> bool:
if self.mode.mock:
self.files_ready = False
return False
if self.antivirus is None:
return self.files_ready
if perf_counter() - self._antivirus_checked_at < 5:
return self.files_ready
async with self._antivirus_check_lock:
if perf_counter() - self._antivirus_checked_at < 5:
return self.files_ready
try:
status = await self.antivirus.status()
maximum_age = timedelta(
hours=self.config.document["antivirus"]["max_signature_age_hours"]
)
if datetime.now(UTC) - status.databases_date > maximum_age:
raise DependencyFailure("KESL databases are stale")
self.signatures_version = status.signatures_version
self.files_ready = True
except DependencyFailure:
self.signatures_version = "unavailable"
self.files_ready = False
self._antivirus_checked_at = perf_counter()
return self.files_ready
def _verdict(
self,
allow: bool,
@@ -222,7 +253,7 @@ class SafetyService:
)
async def _check_file(self, request: FileCheck, digest: bytes) -> Verdict | Pending:
if not self.files_ready:
if not await self.refresh_antivirus():
raise CapabilityUnavailable("files")
rule = validate_metadata(
request.attachment,
@@ -274,7 +305,7 @@ class SafetyService:
declared_checksum=request.attachment.checksum,
rules_version=self.config.rules_version,
detector_version=self.config.detector.version,
scanner_engine="clamav",
scanner_engine="kesl",
signatures_version=self.signatures_version,
origin_trace_id=trace_id,
origin_span_id=span_id,
VM2_services/codebase/services/message-safety/app/settings.py
+4 -2
View File
@@ -32,8 +32,10 @@ class BootstrapSettings(BaseSettings):
default=5, ge=1, le=32, alias="MESSAGE_SAFETY_WORKER_CONCURRENCY"
)
dns_resolvers: str = Field(default="", alias="MESSAGE_SAFETY_DNS_RESOLVERS")
clamav_host: str = Field(default="clamd", alias="MESSAGE_SAFETY_CLAMAV_HOST")
clamav_port: int = Field(default=3310, ge=1, le=65535, alias="MESSAGE_SAFETY_CLAMAV_PORT")
antivirus_socket: Path = Field(
default=Path("/run/han-kesl/scan.sock"),
alias="MESSAGE_SAFETY_ANTIVIRUS_SOCKET",
)
s3_endpoint_url: str = Field(alias="SELECTEL_S3_ENDPOINT_URL")
s3_bucket: str = Field(alias="SELECTEL_S3_BUCKET_QUARANTINE")
artifacts_dir: Path = Field(
VM2_services/codebase/services/message-safety/app/worker.py
+14 -7
View File
@@ -12,8 +12,9 @@ from app.config import validate_config
from app.contracts import Attachment
from app.db import FileVerdictCache, SafetyAudit, engine_and_sessions
from app.file_pipeline import (
ClamAvInstream,
Antivirus,
DependencyFailure,
KeslSocketScanner,
ObjectChanged,
collect_and_hash,
detect_format,
@@ -34,7 +35,7 @@ tracer = trace.get_tracer("message-safety.worker")
class Worker:
def __init__(
self, repository: Repository, reader: S3VersionReader, antivirus: ClamAvInstream, artifacts
self, repository: Repository, reader: S3VersionReader, antivirus: Antivirus, artifacts
) -> None:
self.repository, self.reader, self.antivirus, self.artifacts = (
repository,
@@ -94,10 +95,15 @@ class Worker:
)
record_dependency("s3", "get_object", "success")
rule = detect_format(body, attachment.mime_type)
signatures_version = task.signatures_version
if not rule:
malware = await self.antivirus.scan(one_chunk(body))
record_dependency("clamav", "scan", "success")
rule = "file.malware_detected" if malware else None
scan = await self.antivirus.scan(
one_chunk(body),
scan_timeout=row.config["antivirus"]["scan_timeout_sec"],
)
signatures_version = scan.signatures_version
record_dependency("antivirus", "scan", "success")
rule = "file.malware_detected" if scan.threat else None
with tracer.start_as_current_span("message_safety.worker.finalize"):
finished = await self.repository.finish(
task.id,
@@ -105,6 +111,7 @@ class Worker:
task.lease_generation,
allow=rule is None,
rule_id=rule or "safety.all_checks_passed",
signatures_version=signatures_version,
)
if finished:
record_worker("allow" if rule is None else "deny", task_age)
@@ -116,7 +123,7 @@ class Worker:
rules_version=task.rules_version,
detector_version=task.detector_version,
scanner_engine=task.scanner_engine,
signatures_version=task.signatures_version,
signatures_version=signatures_version,
verdict="allow" if rule is None else "deny",
rule_id=rule or "safety.all_checks_passed",
reason_code=None if rule is None else "message_blocked",
@@ -192,7 +199,7 @@ async def serve() -> None:
settings.s3_access_key.get_secret_value(),
settings.s3_secret_key.get_secret_value(),
),
ClamAvInstream(settings.clamav_host, settings.clamav_port),
KeslSocketScanner(settings.antivirus_socket),
settings.artifacts_dir,
)
async with asyncio.TaskGroup() as group: