Внедрение KESL на ВМ2 + замена CLAMAV на KESL
This commit is contained in:
@@ -30,8 +30,16 @@ message-safety-config activate --version 1 --approved-by security-owner
|
||||
|
||||
`docker-compose.fragment.yml` is an include fragment for the root VM2 Compose. It publishes no
|
||||
host port, runs API and worker as UID 10001 with a read-only filesystem, drops all capabilities,
|
||||
and mounts only service-specific secret files. The root project owns networks/secrets and the
|
||||
root-owned emergency mode file.
|
||||
and mounts only service-specific secret files plus the broker Unix socket
|
||||
`/run/han-kesl/scan.sock` into the worker. The root project owns networks/secrets and the
|
||||
root-owned emergency mode file. KESL 12.4 standalone and its root-owned fail-closed broker are
|
||||
host services, not Compose services; `clamd` and `freshclam` are absent from Compose.
|
||||
|
||||
The broker invokes the fixed host command `kesl-control --scan-file --action Inform`. A clean
|
||||
result may continue to allow, an infected result denies, and scanner errors, unknown output,
|
||||
timeouts or a stale KESL database remain retryable and eventually return `503` rather than
|
||||
allowing content. Runtime records `scanner_engine=kesl`; `signatures_version` is the hash of KESL
|
||||
version plus database date. KESL database updates run hourly under the operator KESL runbook.
|
||||
|
||||
## External release gates
|
||||
|
||||
@@ -41,8 +49,10 @@ target environment verifies them:
|
||||
- Selectel S3 supports version-specific `GetObject`, signed conditional ETag behavior, bucket
|
||||
versioning, checksum metadata, virtual-host addressing and a read-only IAM policy without
|
||||
list/write/delete.
|
||||
- ClamAV engine/signature metadata is supplied to readiness and task cache keys; freshclam
|
||||
activate/reload, signature-age alarms and clean/EICAR/malformed corpora pass on VM2.
|
||||
- KESL version/database date is supplied through the broker to readiness and task cache keys;
|
||||
hourly update, stale-database alarms and clean/EICAR/malformed corpora pass on VM2.
|
||||
- The broker is a custom integration: exact `kesl-control` output/exit semantics, socket
|
||||
permissions, cleanup and throughput must pass gates on the target VM2 with KESL 12.4.
|
||||
- HEIF native decoding and PDF parser sandbox resource limits pass the approved corpus. The
|
||||
in-process detector is bounded by 5 MiB and validates active/encrypted PDF markers, but OS-level
|
||||
CPU/memory/wall-time isolation must be enforced by the worker container and target runtime.
|
||||
|
||||
@@ -96,6 +96,7 @@ def create_app(service: SafetyService, token: str) -> FastAPI:
|
||||
|
||||
@app.get("/health/ready")
|
||||
async def ready() -> JSONResponse:
|
||||
await service.refresh_antivirus()
|
||||
mode = "mock" if service.mode.mock else "standard"
|
||||
components = {
|
||||
"postgres": "ok",
|
||||
|
||||
@@ -2,9 +2,9 @@
|
||||
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"required": ["schema_version", "rules_bundle_ref", "detector_manifest_ref", "task", "rate", "retention", "cache", "link", "clamav", "file_policy"],
|
||||
"required": ["schema_version", "rules_bundle_ref", "detector_manifest_ref", "task", "rate", "retention", "cache", "link", "antivirus", "file_policy"],
|
||||
"properties": {
|
||||
"schema_version": {"const": 1},
|
||||
"schema_version": {"const": 2},
|
||||
"rules_bundle_ref": {"type": "string", "pattern": "^rules-[0-9]{4}-[0-9]{2}-[0-9]{2}$"},
|
||||
"detector_manifest_ref": {"const": "detector-2026-08-03"},
|
||||
"task": {
|
||||
@@ -48,9 +48,13 @@
|
||||
"pipeline_timeout_sec": {"type": "number", "exclusiveMinimum": 0, "maximum": 5}
|
||||
}
|
||||
},
|
||||
"clamav": {
|
||||
"type": "object", "additionalProperties": false, "required": ["scan_timeout_sec", "max_signature_age_hours"],
|
||||
"properties": {"scan_timeout_sec": {"type": "integer", "minimum": 1, "maximum": 120}, "max_signature_age_hours": {"type": "integer", "minimum": 1, "maximum": 720}}
|
||||
"antivirus": {
|
||||
"type": "object", "additionalProperties": false, "required": ["engine", "scan_timeout_sec", "max_signature_age_hours"],
|
||||
"properties": {
|
||||
"engine": {"const": "kesl"},
|
||||
"scan_timeout_sec": {"type": "integer", "minimum": 1, "maximum": 300},
|
||||
"max_signature_age_hours": {"type": "integer", "minimum": 1, "maximum": 720}
|
||||
}
|
||||
},
|
||||
"file_policy": {
|
||||
"type": "object", "additionalProperties": false, "required": ["enabled_mime_types", "max_size_bytes"],
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
schema_version: 1
|
||||
schema_version: 2
|
||||
rules_bundle_ref: rules-2026-01-01
|
||||
detector_manifest_ref: detector-2026-08-03
|
||||
task:
|
||||
@@ -21,8 +21,9 @@ link:
|
||||
url_max_length: 2048
|
||||
dns_lookup_timeout_sec: 1
|
||||
pipeline_timeout_sec: 2
|
||||
clamav:
|
||||
scan_timeout_sec: 45
|
||||
antivirus:
|
||||
engine: kesl
|
||||
scan_timeout_sec: 60
|
||||
max_signature_age_hours: 240
|
||||
file_policy:
|
||||
enabled_mime_types:
|
||||
|
||||
@@ -7,6 +7,8 @@ import json
|
||||
import struct
|
||||
from collections.abc import AsyncIterator
|
||||
from dataclasses import dataclass
|
||||
from datetime import UTC, datetime
|
||||
from email.utils import parsedate_to_datetime
|
||||
from pathlib import Path
|
||||
from typing import Protocol
|
||||
|
||||
@@ -33,7 +35,24 @@ class ObjectReader(Protocol):
|
||||
|
||||
|
||||
class Antivirus(Protocol):
|
||||
async def scan(self, chunks: AsyncIterator[bytes]) -> str | None: ...
|
||||
async def scan(
|
||||
self, chunks: AsyncIterator[bytes], *, scan_timeout: float
|
||||
) -> AntivirusScanResult: ...
|
||||
|
||||
async def status(self) -> AntivirusStatus: ...
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class AntivirusScanResult:
|
||||
threat: str | None
|
||||
signatures_version: str
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class AntivirusStatus:
|
||||
engine_version: str
|
||||
signatures_version: str
|
||||
databases_date: datetime
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
@@ -119,65 +138,101 @@ async def collect_and_hash(
|
||||
return bytes(body), digest.digest()
|
||||
|
||||
|
||||
class ClamAvInstream:
|
||||
def __init__(self, host: str, port: int, timeout: float = 45.0) -> None:
|
||||
self.host, self.port, self.timeout = host, port, timeout
|
||||
class KeslSocketScanner:
|
||||
MAX_HEADER = 4096
|
||||
MAX_RESPONSE = 16 * 1024
|
||||
|
||||
async def scan(self, chunks: AsyncIterator[bytes]) -> str | None:
|
||||
def __init__(self, socket_path: Path, timeout: float = 60.0) -> None:
|
||||
self.socket_path = socket_path
|
||||
self.timeout = timeout
|
||||
|
||||
async def _request(
|
||||
self,
|
||||
header: dict[str, object],
|
||||
body: bytes = b"",
|
||||
*,
|
||||
request_timeout: float | None = None,
|
||||
) -> dict[str, object]:
|
||||
encoded = json.dumps(header, separators=(",", ":")).encode()
|
||||
if len(encoded) > self.MAX_HEADER:
|
||||
raise DependencyFailure("KESL request header is too large")
|
||||
|
||||
async def operation() -> dict[str, object]:
|
||||
reader, writer = await asyncio.open_unix_connection(str(self.socket_path))
|
||||
try:
|
||||
writer.write(struct.pack(">I", len(encoded)) + encoded + body)
|
||||
await writer.drain()
|
||||
(size,) = struct.unpack(">I", await reader.readexactly(4))
|
||||
if size < 2 or size > self.MAX_RESPONSE:
|
||||
raise DependencyFailure("invalid KESL broker response size")
|
||||
value = json.loads((await reader.readexactly(size)).decode())
|
||||
if not isinstance(value, dict):
|
||||
raise DependencyFailure("invalid KESL broker response")
|
||||
return value
|
||||
finally:
|
||||
writer.close()
|
||||
await writer.wait_closed()
|
||||
|
||||
try:
|
||||
return await asyncio.wait_for(operation(), request_timeout or self.timeout)
|
||||
except (OSError, TimeoutError, asyncio.IncompleteReadError, json.JSONDecodeError) as exc:
|
||||
raise DependencyFailure("KESL unavailable") from exc
|
||||
|
||||
async def scan(
|
||||
self, chunks: AsyncIterator[bytes], *, scan_timeout: float
|
||||
) -> AntivirusScanResult:
|
||||
body = bytearray()
|
||||
async for chunk in chunks:
|
||||
body.extend(chunk)
|
||||
with tracer.start_as_current_span(
|
||||
"message_safety.clamav.scan",
|
||||
attributes={"server.address.type": "clamav"},
|
||||
"message_safety.antivirus.scan",
|
||||
attributes={"message_safety.antivirus.engine": "kesl"},
|
||||
):
|
||||
return await self._scan(chunks)
|
||||
|
||||
async def _scan(self, chunks: AsyncIterator[bytes]) -> str | None:
|
||||
async def operation() -> str | None:
|
||||
reader, writer = await asyncio.open_connection(self.host, self.port)
|
||||
try:
|
||||
writer.write(b"zINSTREAM\0")
|
||||
async for chunk in chunks:
|
||||
writer.write(struct.pack(">I", len(chunk)) + chunk)
|
||||
await writer.drain()
|
||||
writer.write(struct.pack(">I", 0))
|
||||
await writer.drain()
|
||||
result = await reader.readuntil(b"\0")
|
||||
text = result.rstrip(b"\0").decode("utf-8", "replace")
|
||||
if text.endswith(" OK"):
|
||||
return None
|
||||
if text.endswith(" FOUND"):
|
||||
return text.rsplit(": ", 1)[-1].removesuffix(" FOUND")
|
||||
raise DependencyFailure("invalid ClamAV response")
|
||||
finally:
|
||||
writer.close()
|
||||
await writer.wait_closed()
|
||||
|
||||
try:
|
||||
return await asyncio.wait_for(operation(), self.timeout)
|
||||
except (OSError, TimeoutError) as exc:
|
||||
raise DependencyFailure("ClamAV unavailable") from exc
|
||||
|
||||
async def signatures_version(self) -> str:
|
||||
with tracer.start_as_current_span("message_safety.clamav.version"):
|
||||
return await self._signatures_version()
|
||||
|
||||
async def _signatures_version(self) -> str:
|
||||
try:
|
||||
reader, writer = await asyncio.wait_for(
|
||||
asyncio.open_connection(self.host, self.port), 2.0
|
||||
value = await self._request(
|
||||
{"op": "scan", "size": len(body)},
|
||||
bytes(body),
|
||||
request_timeout=scan_timeout,
|
||||
)
|
||||
if value.get("status") != "completed" or value.get("verdict") not in {
|
||||
"clean",
|
||||
"infected",
|
||||
}:
|
||||
raise DependencyFailure("KESL scan failed")
|
||||
signatures = value.get("signatures_version")
|
||||
if not isinstance(signatures, str) or not signatures.startswith("sha256:"):
|
||||
raise DependencyFailure("KESL signatures version is missing")
|
||||
threat = value.get("threat")
|
||||
if value["verdict"] == "infected" and not isinstance(threat, str):
|
||||
raise DependencyFailure("KESL infected verdict has no threat")
|
||||
return AntivirusScanResult(
|
||||
threat=threat if isinstance(threat, str) else None,
|
||||
signatures_version=signatures,
|
||||
)
|
||||
|
||||
async def status(self) -> AntivirusStatus:
|
||||
with tracer.start_as_current_span(
|
||||
"message_safety.antivirus.status",
|
||||
attributes={"message_safety.antivirus.engine": "kesl"},
|
||||
):
|
||||
value = await self._request({"op": "status"})
|
||||
if value.get("status") != "ready":
|
||||
raise DependencyFailure("KESL is not ready")
|
||||
try:
|
||||
database_date = datetime.fromisoformat(
|
||||
str(value["databases_date"]).replace("Z", "+00:00")
|
||||
)
|
||||
except ValueError:
|
||||
try:
|
||||
writer.write(b"zVERSION\0")
|
||||
await writer.drain()
|
||||
raw = await asyncio.wait_for(reader.readuntil(b"\0"), 2.0)
|
||||
finally:
|
||||
writer.close()
|
||||
await writer.wait_closed()
|
||||
except (OSError, TimeoutError) as exc:
|
||||
raise DependencyFailure("ClamAV unavailable") from exc
|
||||
value = raw.rstrip(b"\0")
|
||||
if not value.startswith(b"ClamAV ") or len(value) > 512:
|
||||
raise DependencyFailure("invalid ClamAV version response")
|
||||
return "sha256:" + hashlib.sha256(value).hexdigest()
|
||||
database_date = parsedate_to_datetime(str(value["databases_date"]))
|
||||
except (TypeError, ValueError) as exc:
|
||||
raise DependencyFailure("invalid KESL database date") from exc
|
||||
if database_date.tzinfo is None:
|
||||
database_date = database_date.replace(tzinfo=UTC)
|
||||
engine = value.get("engine_version")
|
||||
signatures = value.get("signatures_version")
|
||||
if not isinstance(engine, str) or not isinstance(signatures, str):
|
||||
raise DependencyFailure("incomplete KESL status")
|
||||
return AntivirusStatus(engine, signatures, database_date.astimezone(UTC))
|
||||
|
||||
|
||||
async def one_chunk(data: bytes) -> AsyncIterator[bytes]:
|
||||
|
||||
@@ -8,7 +8,7 @@ from app.adapters import TrustedDnsResolver
|
||||
from app.api import create_app
|
||||
from app.config import ActiveConfig, validate_config
|
||||
from app.db import engine_and_sessions
|
||||
from app.file_pipeline import ClamAvInstream, DependencyFailure
|
||||
from app.file_pipeline import DependencyFailure, KeslSocketScanner
|
||||
from app.repository import Repository
|
||||
from app.service import SafetyService
|
||||
from app.settings import BootstrapSettings, EmergencyMode
|
||||
@@ -29,13 +29,14 @@ async def build_runtime() -> tuple[object, object]:
|
||||
resolver = TrustedDnsResolver(
|
||||
[item.strip() for item in settings.dns_resolvers.split(",") if item.strip()]
|
||||
)
|
||||
clamav = ClamAvInstream(settings.clamav_host, settings.clamav_port)
|
||||
antivirus = KeslSocketScanner(settings.antivirus_socket)
|
||||
if mode.mock:
|
||||
signatures_version = "unavailable"
|
||||
files_ready = False
|
||||
else:
|
||||
try:
|
||||
signatures_version = await clamav.signatures_version()
|
||||
status = await antivirus.status()
|
||||
signatures_version = status.signatures_version
|
||||
files_ready = True
|
||||
except DependencyFailure:
|
||||
signatures_version = "unavailable"
|
||||
@@ -47,6 +48,7 @@ async def build_runtime() -> tuple[object, object]:
|
||||
resolver,
|
||||
files_ready=files_ready,
|
||||
signatures_version=signatures_version,
|
||||
antivirus=antivirus,
|
||||
)
|
||||
app = create_app(service, settings.service_token.get_secret_value())
|
||||
instrument_fastapi(app)
|
||||
|
||||
@@ -90,7 +90,7 @@ class Repository:
|
||||
FileVerdictCache.config_version == config.version,
|
||||
FileVerdictCache.rules_version == config.rules_version,
|
||||
FileVerdictCache.detector_version == config.detector.version,
|
||||
FileVerdictCache.scanner_engine == "clamav",
|
||||
FileVerdictCache.scanner_engine == "kesl",
|
||||
FileVerdictCache.signatures_version == signatures_version,
|
||||
FileVerdictCache.expires_at > func.now(),
|
||||
)
|
||||
@@ -262,9 +262,29 @@ class Repository:
|
||||
return result.rowcount == 1
|
||||
|
||||
async def finish(
|
||||
self, task_id: uuid.UUID, owner: str, generation: int, *, allow: bool, rule_id: str
|
||||
self,
|
||||
task_id: uuid.UUID,
|
||||
owner: str,
|
||||
generation: int,
|
||||
*,
|
||||
allow: bool,
|
||||
rule_id: str,
|
||||
signatures_version: str | None = None,
|
||||
) -> bool:
|
||||
now = datetime.now(UTC)
|
||||
values = {
|
||||
"status": TaskStatus.allowed if allow else TaskStatus.denied,
|
||||
"verdict": "allow" if allow else "deny",
|
||||
"rule_id": rule_id,
|
||||
"reason_code": None if allow else "message_blocked",
|
||||
"finished_at": now,
|
||||
"purge_after": now + timedelta(days=30),
|
||||
"updated_at": now,
|
||||
"lease_owner": None,
|
||||
"lease_until": None,
|
||||
}
|
||||
if signatures_version is not None:
|
||||
values["signatures_version"] = signatures_version
|
||||
async with self.sessions.begin() as session:
|
||||
result = await session.execute(
|
||||
update(SafetyTask)
|
||||
@@ -275,17 +295,7 @@ class Repository:
|
||||
SafetyTask.lease_generation == generation,
|
||||
SafetyTask.lease_until > func.now(),
|
||||
)
|
||||
.values(
|
||||
status=TaskStatus.allowed if allow else TaskStatus.denied,
|
||||
verdict="allow" if allow else "deny",
|
||||
rule_id=rule_id,
|
||||
reason_code=None if allow else "message_blocked",
|
||||
finished_at=now,
|
||||
purge_after=now + timedelta(days=30),
|
||||
updated_at=now,
|
||||
lease_owner=None,
|
||||
lease_until=None,
|
||||
)
|
||||
.values(**values)
|
||||
)
|
||||
if result.rowcount == 1:
|
||||
await session.execute(
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import uuid
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from time import perf_counter
|
||||
@@ -14,7 +15,7 @@ from app.db import (
|
||||
TaskStatus,
|
||||
TextRulesCache,
|
||||
)
|
||||
from app.file_pipeline import validate_metadata
|
||||
from app.file_pipeline import Antivirus, DependencyFailure, validate_metadata
|
||||
from app.fingerprint import fingerprint
|
||||
from app.normalization import normalize_text
|
||||
from app.rate_limit import ConservativeRateLimiter
|
||||
@@ -52,6 +53,7 @@ class SafetyService:
|
||||
links_ready: bool = True,
|
||||
files_ready: bool = True,
|
||||
signatures_version: str = "unverified",
|
||||
antivirus: Antivirus | None = None,
|
||||
) -> None:
|
||||
self.repository = repository
|
||||
self.config = config
|
||||
@@ -60,11 +62,40 @@ class SafetyService:
|
||||
self.links_ready = links_ready
|
||||
self.files_ready = files_ready
|
||||
self.signatures_version = signatures_version
|
||||
self.antivirus = antivirus
|
||||
self._antivirus_checked_at = 0.0
|
||||
self._antivirus_check_lock = asyncio.Lock()
|
||||
self.rate_limiter = ConservativeRateLimiter(
|
||||
config.document["rate"]["text_rps"], config.document["rate"]["file_rps"]
|
||||
)
|
||||
record_runtime_state("mock" if mode.mock else "standard", config.version)
|
||||
|
||||
async def refresh_antivirus(self) -> bool:
|
||||
if self.mode.mock:
|
||||
self.files_ready = False
|
||||
return False
|
||||
if self.antivirus is None:
|
||||
return self.files_ready
|
||||
if perf_counter() - self._antivirus_checked_at < 5:
|
||||
return self.files_ready
|
||||
async with self._antivirus_check_lock:
|
||||
if perf_counter() - self._antivirus_checked_at < 5:
|
||||
return self.files_ready
|
||||
try:
|
||||
status = await self.antivirus.status()
|
||||
maximum_age = timedelta(
|
||||
hours=self.config.document["antivirus"]["max_signature_age_hours"]
|
||||
)
|
||||
if datetime.now(UTC) - status.databases_date > maximum_age:
|
||||
raise DependencyFailure("KESL databases are stale")
|
||||
self.signatures_version = status.signatures_version
|
||||
self.files_ready = True
|
||||
except DependencyFailure:
|
||||
self.signatures_version = "unavailable"
|
||||
self.files_ready = False
|
||||
self._antivirus_checked_at = perf_counter()
|
||||
return self.files_ready
|
||||
|
||||
def _verdict(
|
||||
self,
|
||||
allow: bool,
|
||||
@@ -222,7 +253,7 @@ class SafetyService:
|
||||
)
|
||||
|
||||
async def _check_file(self, request: FileCheck, digest: bytes) -> Verdict | Pending:
|
||||
if not self.files_ready:
|
||||
if not await self.refresh_antivirus():
|
||||
raise CapabilityUnavailable("files")
|
||||
rule = validate_metadata(
|
||||
request.attachment,
|
||||
@@ -274,7 +305,7 @@ class SafetyService:
|
||||
declared_checksum=request.attachment.checksum,
|
||||
rules_version=self.config.rules_version,
|
||||
detector_version=self.config.detector.version,
|
||||
scanner_engine="clamav",
|
||||
scanner_engine="kesl",
|
||||
signatures_version=self.signatures_version,
|
||||
origin_trace_id=trace_id,
|
||||
origin_span_id=span_id,
|
||||
|
||||
@@ -32,8 +32,10 @@ class BootstrapSettings(BaseSettings):
|
||||
default=5, ge=1, le=32, alias="MESSAGE_SAFETY_WORKER_CONCURRENCY"
|
||||
)
|
||||
dns_resolvers: str = Field(default="", alias="MESSAGE_SAFETY_DNS_RESOLVERS")
|
||||
clamav_host: str = Field(default="clamd", alias="MESSAGE_SAFETY_CLAMAV_HOST")
|
||||
clamav_port: int = Field(default=3310, ge=1, le=65535, alias="MESSAGE_SAFETY_CLAMAV_PORT")
|
||||
antivirus_socket: Path = Field(
|
||||
default=Path("/run/han-kesl/scan.sock"),
|
||||
alias="MESSAGE_SAFETY_ANTIVIRUS_SOCKET",
|
||||
)
|
||||
s3_endpoint_url: str = Field(alias="SELECTEL_S3_ENDPOINT_URL")
|
||||
s3_bucket: str = Field(alias="SELECTEL_S3_BUCKET_QUARANTINE")
|
||||
artifacts_dir: Path = Field(
|
||||
|
||||
@@ -12,8 +12,9 @@ from app.config import validate_config
|
||||
from app.contracts import Attachment
|
||||
from app.db import FileVerdictCache, SafetyAudit, engine_and_sessions
|
||||
from app.file_pipeline import (
|
||||
ClamAvInstream,
|
||||
Antivirus,
|
||||
DependencyFailure,
|
||||
KeslSocketScanner,
|
||||
ObjectChanged,
|
||||
collect_and_hash,
|
||||
detect_format,
|
||||
@@ -34,7 +35,7 @@ tracer = trace.get_tracer("message-safety.worker")
|
||||
|
||||
class Worker:
|
||||
def __init__(
|
||||
self, repository: Repository, reader: S3VersionReader, antivirus: ClamAvInstream, artifacts
|
||||
self, repository: Repository, reader: S3VersionReader, antivirus: Antivirus, artifacts
|
||||
) -> None:
|
||||
self.repository, self.reader, self.antivirus, self.artifacts = (
|
||||
repository,
|
||||
@@ -94,10 +95,15 @@ class Worker:
|
||||
)
|
||||
record_dependency("s3", "get_object", "success")
|
||||
rule = detect_format(body, attachment.mime_type)
|
||||
signatures_version = task.signatures_version
|
||||
if not rule:
|
||||
malware = await self.antivirus.scan(one_chunk(body))
|
||||
record_dependency("clamav", "scan", "success")
|
||||
rule = "file.malware_detected" if malware else None
|
||||
scan = await self.antivirus.scan(
|
||||
one_chunk(body),
|
||||
scan_timeout=row.config["antivirus"]["scan_timeout_sec"],
|
||||
)
|
||||
signatures_version = scan.signatures_version
|
||||
record_dependency("antivirus", "scan", "success")
|
||||
rule = "file.malware_detected" if scan.threat else None
|
||||
with tracer.start_as_current_span("message_safety.worker.finalize"):
|
||||
finished = await self.repository.finish(
|
||||
task.id,
|
||||
@@ -105,6 +111,7 @@ class Worker:
|
||||
task.lease_generation,
|
||||
allow=rule is None,
|
||||
rule_id=rule or "safety.all_checks_passed",
|
||||
signatures_version=signatures_version,
|
||||
)
|
||||
if finished:
|
||||
record_worker("allow" if rule is None else "deny", task_age)
|
||||
@@ -116,7 +123,7 @@ class Worker:
|
||||
rules_version=task.rules_version,
|
||||
detector_version=task.detector_version,
|
||||
scanner_engine=task.scanner_engine,
|
||||
signatures_version=task.signatures_version,
|
||||
signatures_version=signatures_version,
|
||||
verdict="allow" if rule is None else "deny",
|
||||
rule_id=rule or "safety.all_checks_passed",
|
||||
reason_code=None if rule is None else "message_blocked",
|
||||
@@ -192,7 +199,7 @@ async def serve() -> None:
|
||||
settings.s3_access_key.get_secret_value(),
|
||||
settings.s3_secret_key.get_secret_value(),
|
||||
),
|
||||
ClamAvInstream(settings.clamav_host, settings.clamav_port),
|
||||
KeslSocketScanner(settings.antivirus_socket),
|
||||
settings.artifacts_dir,
|
||||
)
|
||||
async with asyncio.TaskGroup() as group:
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from uuid import uuid4
|
||||
|
||||
import httpx
|
||||
@@ -7,6 +8,7 @@ import pytest
|
||||
|
||||
from app.api import create_app
|
||||
from app.db import TaskStatus
|
||||
from app.file_pipeline import AntivirusStatus
|
||||
from app.repository import ConflictError
|
||||
from app.service import SafetyService
|
||||
from app.settings import EmergencyMode
|
||||
@@ -54,6 +56,15 @@ class ForbiddenResolver:
|
||||
raise AssertionError("MOCK must not call DNS")
|
||||
|
||||
|
||||
class StaleAntivirus:
|
||||
async def status(self):
|
||||
return AntivirusStatus(
|
||||
"12.4",
|
||||
"sha256:" + "a" * 64,
|
||||
datetime.now(UTC) - timedelta(hours=241),
|
||||
)
|
||||
|
||||
|
||||
def body(kind: str, message_id=None) -> dict:
|
||||
value = {
|
||||
"message_id": str(message_id or uuid4()),
|
||||
@@ -188,3 +199,27 @@ async def test_final_task_response_keeps_task_config_snapshot(active_config) ->
|
||||
|
||||
assert final.status_code == 200
|
||||
assert final.json()["config_version"] == 99
|
||||
|
||||
|
||||
async def test_stale_kesl_databases_disable_only_files(active_config) -> None:
|
||||
repo = FakeRepository()
|
||||
service = SafetyService(
|
||||
repo,
|
||||
active_config,
|
||||
EmergencyMode(),
|
||||
ForbiddenResolver(),
|
||||
antivirus=StaleAntivirus(),
|
||||
)
|
||||
async with httpx.AsyncClient(
|
||||
transport=httpx.ASGITransport(app=create_app(service, "secret")),
|
||||
base_url="http://test",
|
||||
headers={"X-Service-Token": "secret"},
|
||||
) as client:
|
||||
file_response = await client.post(
|
||||
"/internal/safety/v2/messages/check", json=body("file")
|
||||
)
|
||||
text_response = await client.post(
|
||||
"/internal/safety/v2/messages/check", json=body("text")
|
||||
)
|
||||
assert file_response.status_code == 503
|
||||
assert text_response.status_code == 200
|
||||
|
||||
@@ -35,12 +35,13 @@ def test_config_cross_field_and_manifest_subset(artifacts: Path) -> None:
|
||||
validate_config(bad, artifacts)
|
||||
|
||||
|
||||
def test_clamav_signature_age_policy_bounds(artifacts: Path) -> None:
|
||||
def test_kesl_signature_age_policy_bounds(artifacts: Path) -> None:
|
||||
document = seed(artifacts)
|
||||
assert document["clamav"]["max_signature_age_hours"] == 240
|
||||
document["clamav"]["max_signature_age_hours"] = 720
|
||||
assert document["antivirus"]["engine"] == "kesl"
|
||||
assert document["antivirus"]["max_signature_age_hours"] == 240
|
||||
document["antivirus"]["max_signature_age_hours"] = 720
|
||||
validate_config(document, artifacts)
|
||||
document["clamav"]["max_signature_age_hours"] = 721
|
||||
document["antivirus"]["max_signature_age_hours"] = 721
|
||||
with pytest.raises(ValidationError):
|
||||
validate_config(document, artifacts)
|
||||
|
||||
|
||||
@@ -1,14 +1,24 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import hashlib
|
||||
import io
|
||||
import json
|
||||
import struct
|
||||
from pathlib import Path
|
||||
from uuid import UUID
|
||||
|
||||
import pytest
|
||||
from PIL import Image
|
||||
|
||||
from app.contracts import Attachment
|
||||
from app.file_pipeline import ObjectChanged, collect_and_hash, detect_format
|
||||
from app.file_pipeline import (
|
||||
KeslSocketScanner,
|
||||
ObjectChanged,
|
||||
collect_and_hash,
|
||||
detect_format,
|
||||
one_chunk,
|
||||
)
|
||||
|
||||
|
||||
def image_bytes(format_name: str) -> bytes:
|
||||
@@ -71,3 +81,65 @@ async def test_authoritative_stream_hash_and_size() -> None:
|
||||
assert body == data and digest == hashlib.sha256(data).digest()
|
||||
with pytest.raises(ObjectChanged):
|
||||
await collect_and_hash(Reader(data), attachment(data, size=len(data) + 1), max_size=100)
|
||||
|
||||
|
||||
class FakeWriter:
|
||||
def __init__(self) -> None:
|
||||
self.request = bytearray()
|
||||
|
||||
def write(self, value: bytes) -> None:
|
||||
self.request.extend(value)
|
||||
|
||||
async def drain(self) -> None:
|
||||
return None
|
||||
|
||||
def close(self) -> None:
|
||||
return None
|
||||
|
||||
async def wait_closed(self) -> None:
|
||||
return None
|
||||
|
||||
|
||||
def framed(value: dict[str, object]):
|
||||
body = json.dumps(value).encode()
|
||||
reader = __import__("asyncio").StreamReader()
|
||||
reader.feed_data(struct.pack(">I", len(body)) + body)
|
||||
reader.feed_eof()
|
||||
return reader
|
||||
|
||||
|
||||
async def test_kesl_socket_clean_infected_and_status(monkeypatch, tmp_path: Path) -> None:
|
||||
responses = [
|
||||
{
|
||||
"status": "completed",
|
||||
"verdict": "clean",
|
||||
"threat": None,
|
||||
"engine_version": "12.4",
|
||||
"signatures_version": "sha256:" + "a" * 64,
|
||||
},
|
||||
{
|
||||
"status": "completed",
|
||||
"verdict": "infected",
|
||||
"threat": "EICAR-Test-File",
|
||||
"engine_version": "12.4",
|
||||
"signatures_version": "sha256:" + "b" * 64,
|
||||
},
|
||||
{
|
||||
"status": "ready",
|
||||
"engine_version": "12.4",
|
||||
"databases_date": "2026-09-07T11:25:00+00:00",
|
||||
"signatures_version": "sha256:" + "c" * 64,
|
||||
},
|
||||
]
|
||||
|
||||
async def connect(_):
|
||||
return framed(responses.pop(0)), FakeWriter()
|
||||
|
||||
monkeypatch.setattr(asyncio, "open_unix_connection", connect, raising=False)
|
||||
scanner = KeslSocketScanner(tmp_path / "scan.sock")
|
||||
clean = await scanner.scan(one_chunk(b"clean"), scan_timeout=1)
|
||||
infected = await scanner.scan(one_chunk(b"eicar"), scan_timeout=1)
|
||||
status = await scanner.status()
|
||||
assert clean.threat is None
|
||||
assert infected.threat == "EICAR-Test-File"
|
||||
assert status.engine_version == "12.4"
|
||||
|
||||
Reference in New Issue
Block a user