mi
845 lines
32 KiB
Python
845 lines
32 KiB
Python
#!/usr/bin/env python3
|
||
"""Самоочищающийся инфраструктурный тест PostgreSQL и S3 для HAN Chat."""
|
||
|
||
from __future__ import annotations
|
||
|
||
import argparse
|
||
import json
|
||
import os
|
||
import re
|
||
import sys
|
||
import uuid
|
||
from dataclasses import dataclass, field
|
||
from pathlib import Path
|
||
from typing import Any, Callable
|
||
from urllib.parse import urlsplit, urlunsplit
|
||
|
||
try:
|
||
import boto3
|
||
import psycopg
|
||
import requests
|
||
from botocore.config import Config as BotoConfig
|
||
from botocore.exceptions import ClientError
|
||
from dotenv import dotenv_values
|
||
from psycopg import sql
|
||
except ImportError as exc: # pragma: no cover - понятная ошибка до запуска тестов
|
||
print(
|
||
f"Не установлена зависимость {exc.name!r}. "
|
||
"Выполните: python3 -m pip install -r requirements.txt",
|
||
file=sys.stderr,
|
||
)
|
||
raise SystemExit(2) from exc
|
||
|
||
|
||
URL_PATTERN = re.compile(r"https?://[^\s]+")
|
||
EXPECTED_ROLES = (
|
||
("han_app", "han_app", "HAN_PG_PASSWORD_HAN_APP"),
|
||
("bitrix_local_app", "bitrix_local", "HAN_PG_PASSWORD_BITRIX"),
|
||
("bitrix_sync_user", "bitrix_sync", "HAN_PG_PASSWORD_BITRIX_SYNC"),
|
||
("message_safety_app", "message_safety", "HAN_PG_PASSWORD_MESSAGE_SAFETY"),
|
||
("keycloak_user", "keycloak", "HAN_PG_PASSWORD_KEYCLOAK"),
|
||
)
|
||
EXPECTED_TABLES = {
|
||
"han_app": (
|
||
"alembic_version",
|
||
"app_settings",
|
||
"dialogs",
|
||
"messages",
|
||
"message_attachments",
|
||
"documents",
|
||
"sync_queue",
|
||
),
|
||
"bitrix_local": (
|
||
"alembic_version",
|
||
"portal_installations",
|
||
"dialog_sessions",
|
||
"inbox_events",
|
||
"outbound_messages",
|
||
),
|
||
"bitrix_sync": ("alembic_version",),
|
||
}
|
||
|
||
|
||
class ConfigError(ValueError):
|
||
"""Ошибка пользовательской конфигурации."""
|
||
|
||
|
||
def as_bool(value: str | None, default: bool = False) -> bool:
|
||
if value is None or not value.strip():
|
||
return default
|
||
normalized = value.strip().lower()
|
||
if normalized in {"1", "true", "yes", "on"}:
|
||
return True
|
||
if normalized in {"0", "false", "no", "off"}:
|
||
return False
|
||
raise ConfigError(f"Ожидалось логическое значение, получено: {value!r}")
|
||
|
||
|
||
def sanitize(value: Any) -> str:
|
||
"""Удаляет секреты и query-параметры URL из диагностического текста."""
|
||
text = str(value)
|
||
text = URL_PATTERN.sub(
|
||
lambda match: urlunsplit(
|
||
(*urlsplit(match.group(0))[:3], "", "")
|
||
),
|
||
text,
|
||
)
|
||
text = re.sub(
|
||
r"(?i)\b(password|secret|access[_-]?key|authorization|token|dsn)"
|
||
r"\s*[:=]\s*[^\s,;]+",
|
||
lambda match: f"{match.group(1)}=<redacted>",
|
||
text,
|
||
)
|
||
return text[:1000]
|
||
|
||
|
||
def require(values: dict[str, str], name: str) -> str:
|
||
value = values.get(name, "").strip()
|
||
if not value:
|
||
raise ConfigError(f"Не задана обязательная переменная {name}")
|
||
return value
|
||
|
||
|
||
@dataclass(frozen=True)
|
||
class PgRole:
|
||
user: str
|
||
schema: str
|
||
password: str = field(repr=False)
|
||
|
||
|
||
@dataclass(frozen=True)
|
||
class Settings:
|
||
pg_host: str
|
||
pg_port: int
|
||
pg_database: str
|
||
pg_sslmode: str
|
||
pg_sslrootcert: str
|
||
pg_roles: tuple[PgRole, ...]
|
||
s3_endpoint: str
|
||
s3_region: str | None
|
||
buckets: dict[str, str]
|
||
s3_access_key: str = field(repr=False)
|
||
s3_secret_key: str = field(repr=False)
|
||
s3_read_access_key: str = field(repr=False)
|
||
s3_read_secret_key: str = field(repr=False)
|
||
public_web_url: str | None
|
||
check_cors: bool
|
||
check_migrations: bool
|
||
timeout_seconds: int
|
||
|
||
@classmethod
|
||
def from_env_file(cls, path: Path) -> "Settings":
|
||
if not path.is_file():
|
||
raise ConfigError(f"Файл настроек не найден: {path}")
|
||
raw = dotenv_values(path)
|
||
values = {
|
||
key: str(value)
|
||
for key, value in raw.items()
|
||
if value is not None
|
||
}
|
||
# Явно экспортированные переменные имеют приоритет над файлом.
|
||
values.update({key: value for key, value in os.environ.items() if key in values})
|
||
|
||
sslmode = values.get("HAN_PG_SSLMODE", "verify-full").strip()
|
||
if sslmode != "verify-full":
|
||
raise ConfigError("HAN_PG_SSLMODE должен быть verify-full")
|
||
ca_path = require(values, "HAN_PG_SSLROOTCERT")
|
||
if not Path(ca_path).is_file():
|
||
raise ConfigError(f"CA-файл PostgreSQL не найден: {ca_path}")
|
||
|
||
endpoint = require(values, "SELECTEL_S3_ENDPOINT_URL")
|
||
parsed_endpoint = urlsplit(endpoint)
|
||
if parsed_endpoint.scheme != "https" or not parsed_endpoint.netloc:
|
||
raise ConfigError("SELECTEL_S3_ENDPOINT_URL должен быть корректным HTTPS URL")
|
||
|
||
roles = tuple(
|
||
PgRole(
|
||
values.get(f"HAN_PG_USER_{env_suffix}", default_user).strip(),
|
||
values.get(f"HAN_PG_SCHEMA_{env_suffix}", default_schema).strip(),
|
||
require(values, password_env),
|
||
)
|
||
for default_user, default_schema, password_env in EXPECTED_ROLES
|
||
for env_suffix in (
|
||
{
|
||
"han_app": "HAN_APP",
|
||
"bitrix_local": "BITRIX",
|
||
"bitrix_sync": "BITRIX_SYNC",
|
||
"message_safety": "MESSAGE_SAFETY",
|
||
"keycloak": "KEYCLOAK",
|
||
}[default_schema],
|
||
)
|
||
)
|
||
timeout = int(values.get("INFRATEST_TIMEOUT_SECONDS", "15"))
|
||
if timeout < 1 or timeout > 300:
|
||
raise ConfigError("INFRATEST_TIMEOUT_SECONDS должен быть от 1 до 300")
|
||
|
||
public_web_url = values.get("PUBLIC_WEB_URL", "").strip() or None
|
||
check_cors = as_bool(values.get("INFRATEST_CHECK_CORS"), True)
|
||
if check_cors and not public_web_url:
|
||
raise ConfigError("Для CORS-проверки задайте PUBLIC_WEB_URL")
|
||
|
||
return cls(
|
||
pg_host=require(values, "HAN_PG_HOST"),
|
||
pg_port=int(values.get("HAN_PG_PORT", "5432")),
|
||
pg_database=require(values, "HAN_PG_DATABASE"),
|
||
pg_sslmode=sslmode,
|
||
pg_sslrootcert=ca_path,
|
||
pg_roles=roles,
|
||
s3_endpoint=endpoint.rstrip("/"),
|
||
s3_region=values.get("SELECTEL_S3_REGION", "").strip() or None,
|
||
buckets={
|
||
"quarantine": require(values, "SELECTEL_S3_BUCKET_QUARANTINE"),
|
||
"attachments": require(values, "SELECTEL_S3_BUCKET_ATTACHMENTS"),
|
||
"documents": require(values, "SELECTEL_S3_BUCKET_DOCUMENTS"),
|
||
},
|
||
s3_access_key=require(values, "SELECTEL_S3_ACCESS_KEY"),
|
||
s3_secret_key=require(values, "SELECTEL_S3_SECRET_KEY"),
|
||
s3_read_access_key=require(
|
||
values, "SELECTEL_S3_QUARANTINE_READ_ACCESS_KEY"
|
||
),
|
||
s3_read_secret_key=require(
|
||
values, "SELECTEL_S3_QUARANTINE_READ_SECRET_KEY"
|
||
),
|
||
public_web_url=public_web_url,
|
||
check_cors=check_cors,
|
||
check_migrations=as_bool(
|
||
values.get("INFRATEST_CHECK_MIGRATIONS"), False
|
||
),
|
||
timeout_seconds=timeout,
|
||
)
|
||
|
||
|
||
@dataclass(frozen=True)
|
||
class Result:
|
||
check: str
|
||
status: str
|
||
detail: str
|
||
|
||
|
||
class Reporter:
|
||
def __init__(self) -> None:
|
||
self.results: list[Result] = []
|
||
|
||
def add(self, check: str, status: str, detail: Any) -> None:
|
||
result = Result(check, status, sanitize(detail))
|
||
self.results.append(result)
|
||
print(f"[{status:4}] {check}: {result.detail}")
|
||
|
||
def run(
|
||
self,
|
||
check: str,
|
||
operation: Callable[[], Any],
|
||
success_detail: str = "OK",
|
||
) -> bool:
|
||
try:
|
||
detail = operation()
|
||
except Exception as exc: # noqa: BLE001 - каждая проверка должна продолжить отчёт
|
||
self.add(check, "FAIL", f"{type(exc).__name__}: {exc}")
|
||
return False
|
||
self.add(check, "PASS", success_detail if detail is None else detail)
|
||
return True
|
||
|
||
def skip(self, check: str, detail: str) -> None:
|
||
self.add(check, "SKIP", detail)
|
||
|
||
@property
|
||
def exit_code(self) -> int:
|
||
return 1 if any(item.status == "FAIL" for item in self.results) else 0
|
||
|
||
def write_json(self, path: Path) -> None:
|
||
path.write_text(
|
||
json.dumps(
|
||
[item.__dict__ for item in self.results],
|
||
ensure_ascii=False,
|
||
indent=2,
|
||
)
|
||
+ "\n",
|
||
encoding="utf-8",
|
||
)
|
||
|
||
|
||
class InfraTest:
|
||
def __init__(self, settings: Settings, reporter: Reporter, mode: str) -> None:
|
||
self.settings = settings
|
||
self.reporter = reporter
|
||
self.mode = mode
|
||
self.run_id = uuid.uuid4().hex
|
||
self.prefix = f"infratest/{self.run_id}"
|
||
self.payload = (
|
||
b"HAN Chat infrastructure test\n"
|
||
+ self.run_id.encode("ascii")
|
||
+ b"\n"
|
||
)
|
||
self.api_s3 = self._s3_client(
|
||
settings.s3_access_key, settings.s3_secret_key
|
||
)
|
||
self.read_s3 = self._s3_client(
|
||
settings.s3_read_access_key, settings.s3_read_secret_key
|
||
)
|
||
self.created_objects: set[tuple[str, str]] = set()
|
||
|
||
def _s3_client(self, access_key: str, secret_key: str):
|
||
kwargs: dict[str, Any] = {
|
||
"service_name": "s3",
|
||
"endpoint_url": self.settings.s3_endpoint,
|
||
"aws_access_key_id": access_key,
|
||
"aws_secret_access_key": secret_key,
|
||
"config": BotoConfig(
|
||
connect_timeout=self.settings.timeout_seconds,
|
||
read_timeout=self.settings.timeout_seconds,
|
||
retries={"max_attempts": 2},
|
||
),
|
||
}
|
||
if self.settings.s3_region:
|
||
kwargs["region_name"] = self.settings.s3_region
|
||
return boto3.client(**kwargs)
|
||
|
||
def _pg_connect(self, role: PgRole):
|
||
return psycopg.connect(
|
||
host=self.settings.pg_host,
|
||
port=self.settings.pg_port,
|
||
dbname=self.settings.pg_database,
|
||
user=role.user,
|
||
password=role.password,
|
||
sslmode=self.settings.pg_sslmode,
|
||
sslrootcert=self.settings.pg_sslrootcert,
|
||
connect_timeout=self.settings.timeout_seconds,
|
||
application_name=f"han-infratest-{self.run_id[:8]}",
|
||
)
|
||
|
||
def run(self) -> None:
|
||
self.run_postgres()
|
||
self.run_s3()
|
||
|
||
def run_postgres(self) -> None:
|
||
print("\nPostgreSQL")
|
||
schemas = tuple(role.schema for role in self.settings.pg_roles)
|
||
for role in self.settings.pg_roles:
|
||
connection_ok = self.reporter.run(
|
||
f"postgres.{role.user}.connect_tls",
|
||
lambda role=role: self._check_pg_connection(role),
|
||
)
|
||
if not connection_ok:
|
||
continue
|
||
self.reporter.run(
|
||
f"postgres.{role.user}.schema_isolation",
|
||
lambda role=role: self._check_schema_isolation(role, schemas),
|
||
)
|
||
if self.mode == "full":
|
||
self.reporter.run(
|
||
f"postgres.{role.user}.ddl_dml_rollback",
|
||
lambda role=role: self._check_pg_write(role),
|
||
)
|
||
else:
|
||
self.reporter.skip(
|
||
f"postgres.{role.user}.ddl_dml_rollback",
|
||
"режим readonly",
|
||
)
|
||
|
||
self.reporter.run("postgres.pgcrypto", self._check_pgcrypto)
|
||
if self.settings.check_migrations:
|
||
self.reporter.run("postgres.migrations", self._check_migrations)
|
||
else:
|
||
self.reporter.skip(
|
||
"postgres.migrations",
|
||
"INFRATEST_CHECK_MIGRATIONS=false",
|
||
)
|
||
|
||
def _check_pg_connection(self, role: PgRole) -> str:
|
||
with self._pg_connect(role) as conn:
|
||
with conn.cursor() as cur:
|
||
cur.execute(
|
||
"""
|
||
SELECT current_user, current_setting('search_path'),
|
||
EXISTS (
|
||
SELECT 1 FROM pg_stat_ssl
|
||
WHERE pid = pg_backend_pid() AND ssl
|
||
)
|
||
"""
|
||
)
|
||
current_user, search_path, tls_active = cur.fetchone()
|
||
if current_user != role.user:
|
||
raise AssertionError(
|
||
f"current_user={current_user!r}, ожидался {role.user!r}"
|
||
)
|
||
if role.schema not in search_path.split(","):
|
||
raise AssertionError(
|
||
f"search_path={search_path!r}, ожидалась {role.schema!r}"
|
||
)
|
||
if not tls_active:
|
||
raise AssertionError("соединение установлено без TLS")
|
||
return f"TLS active, search_path={search_path}"
|
||
|
||
def _check_schema_isolation(
|
||
self, role: PgRole, all_schemas: tuple[str, ...]
|
||
) -> str:
|
||
with self._pg_connect(role) as conn:
|
||
with conn.cursor() as cur:
|
||
cur.execute(
|
||
"""
|
||
SELECT EXISTS (
|
||
SELECT 1 FROM information_schema.schemata
|
||
WHERE schema_name = %s
|
||
), has_schema_privilege(current_user, %s, 'USAGE')
|
||
""",
|
||
(role.schema, role.schema),
|
||
)
|
||
schema_exists, own_usage = cur.fetchone()
|
||
if not schema_exists or not own_usage:
|
||
raise AssertionError(
|
||
f"нет схемы или USAGE для {role.schema}"
|
||
)
|
||
unexpected = []
|
||
for other in all_schemas:
|
||
if other == role.schema:
|
||
continue
|
||
cur.execute(
|
||
"SELECT has_schema_privilege(current_user, %s, 'USAGE')",
|
||
(other,),
|
||
)
|
||
if cur.fetchone()[0]:
|
||
unexpected.append(other)
|
||
if unexpected:
|
||
raise AssertionError(
|
||
"обнаружен USAGE на чужие схемы: " + ", ".join(unexpected)
|
||
)
|
||
return f"доступ только к схеме {role.schema}"
|
||
|
||
def _check_pg_write(self, role: PgRole) -> str:
|
||
table_name = f"_infratest_{self.run_id}"
|
||
with self._pg_connect(role) as conn:
|
||
try:
|
||
with conn.cursor() as cur:
|
||
cur.execute(
|
||
sql.SQL("CREATE TABLE {}.{} (id integer PRIMARY KEY, value text)")
|
||
.format(sql.Identifier(role.schema), sql.Identifier(table_name))
|
||
)
|
||
cur.execute(
|
||
sql.SQL("INSERT INTO {}.{} VALUES (%s, %s)").format(
|
||
sql.Identifier(role.schema), sql.Identifier(table_name)
|
||
),
|
||
(1, "ok"),
|
||
)
|
||
cur.execute(
|
||
sql.SQL("SELECT value FROM {}.{} WHERE id = %s").format(
|
||
sql.Identifier(role.schema), sql.Identifier(table_name)
|
||
),
|
||
(1,),
|
||
)
|
||
if cur.fetchone()[0] != "ok":
|
||
raise AssertionError("прочитано неожиданное значение")
|
||
finally:
|
||
conn.rollback()
|
||
with conn.cursor() as cur:
|
||
cur.execute("SELECT to_regclass(%s)", (f"{role.schema}.{table_name}",))
|
||
if cur.fetchone()[0] is not None:
|
||
raise AssertionError("таблица сохранилась после ROLLBACK")
|
||
return "CREATE/INSERT/SELECT успешны, DDL откачен"
|
||
|
||
def _check_pgcrypto(self) -> str:
|
||
role = next(item for item in self.settings.pg_roles if item.schema == "han_app")
|
||
with self._pg_connect(role) as conn:
|
||
with conn.cursor() as cur:
|
||
cur.execute(
|
||
"SELECT EXISTS (SELECT 1 FROM pg_extension WHERE extname='pgcrypto')"
|
||
)
|
||
if not cur.fetchone()[0]:
|
||
raise AssertionError("расширение pgcrypto не установлено")
|
||
return "расширение pgcrypto установлено"
|
||
|
||
def _check_migrations(self) -> str:
|
||
role_by_schema = {role.schema: role for role in self.settings.pg_roles}
|
||
missing: list[str] = []
|
||
revisions: dict[str, str] = {}
|
||
for schema, tables in EXPECTED_TABLES.items():
|
||
with self._pg_connect(role_by_schema[schema]) as conn:
|
||
with conn.cursor() as cur:
|
||
for table in tables:
|
||
cur.execute("SELECT to_regclass(%s)", (f"{schema}.{table}",))
|
||
if cur.fetchone()[0] is None:
|
||
missing.append(f"{schema}.{table}")
|
||
cur.execute(
|
||
sql.SQL("SELECT version_num FROM {}.alembic_version LIMIT 1")
|
||
.format(sql.Identifier(schema))
|
||
)
|
||
row = cur.fetchone()
|
||
revisions[schema] = row[0] if row else "<empty>"
|
||
if missing:
|
||
raise AssertionError("нет таблиц: " + ", ".join(missing))
|
||
if revisions.get("han_app") != "0001_initial":
|
||
raise AssertionError(
|
||
f"han_app revision={revisions.get('han_app')!r}, ожидался '0001_initial'"
|
||
)
|
||
return "таблицы и Alembic revisions присутствуют"
|
||
|
||
def run_s3(self) -> None:
|
||
print("\nSelectel S3")
|
||
reachable: dict[str, bool] = {}
|
||
for logical_name, bucket in self.settings.buckets.items():
|
||
reachable[logical_name] = self.reporter.run(
|
||
f"s3.{logical_name}.head_bucket",
|
||
lambda bucket=bucket: self.api_s3.head_bucket(Bucket=bucket),
|
||
f"бакет {bucket} доступен",
|
||
)
|
||
|
||
self.reporter.run(
|
||
"s3.quarantine_read_key.boundaries",
|
||
self._check_read_key_bucket_boundaries,
|
||
)
|
||
if self.mode != "full":
|
||
for name in (
|
||
"direct_vm_operations",
|
||
"presigned_put_get",
|
||
"copy_promote",
|
||
"cors",
|
||
"anonymous_access",
|
||
"quarantine_read_key.object_permissions",
|
||
):
|
||
self.reporter.skip(f"s3.{name}", "режим readonly")
|
||
return
|
||
if not all(reachable.values()):
|
||
self.reporter.add(
|
||
"s3.full_operations",
|
||
"FAIL",
|
||
"полные проверки невозможны: не все бакеты доступны",
|
||
)
|
||
return
|
||
|
||
try:
|
||
direct_keys: dict[str, str] = {}
|
||
for logical_name, bucket in self.settings.buckets.items():
|
||
key = f"{self.prefix}/direct-{logical_name}.bin"
|
||
direct_keys[logical_name] = key
|
||
self.reporter.run(
|
||
f"s3.{logical_name}.direct_vm_operations",
|
||
lambda bucket=bucket, key=key: self._check_direct_object(
|
||
bucket, key
|
||
),
|
||
)
|
||
|
||
presigned_key = f"{self.prefix}/presigned.bin"
|
||
presigned_ok = self.reporter.run(
|
||
"s3.quarantine.presigned_put_get",
|
||
lambda: self._check_presigned(presigned_key),
|
||
)
|
||
if presigned_ok:
|
||
promoted_key = f"{self.prefix}/promoted.bin"
|
||
self.reporter.run(
|
||
"s3.quarantine_to_attachments.copy_promote",
|
||
lambda: self._check_promote(presigned_key, promoted_key),
|
||
)
|
||
|
||
privacy_key = direct_keys["documents"]
|
||
self.reporter.run(
|
||
"s3.documents.anonymous_access_denied",
|
||
lambda: self._check_anonymous_denied(
|
||
self.settings.buckets["documents"], privacy_key
|
||
),
|
||
)
|
||
self.reporter.run(
|
||
"s3.quarantine_read_key.object_permissions",
|
||
lambda: self._check_read_key_object_permissions(
|
||
direct_keys["quarantine"],
|
||
direct_keys["attachments"],
|
||
direct_keys["documents"],
|
||
),
|
||
)
|
||
if self.settings.check_cors:
|
||
cors_key = f"{self.prefix}/cors.bin"
|
||
self.reporter.run(
|
||
"s3.quarantine.cors_preflight",
|
||
lambda: self._check_cors(cors_key),
|
||
)
|
||
else:
|
||
self.reporter.skip(
|
||
"s3.quarantine.cors_preflight",
|
||
"INFRATEST_CHECK_CORS=false",
|
||
)
|
||
finally:
|
||
self._cleanup()
|
||
|
||
def _check_direct_object(self, bucket: str, key: str) -> str:
|
||
self.api_s3.put_object(
|
||
Bucket=bucket,
|
||
Key=key,
|
||
Body=self.payload,
|
||
ContentType="application/octet-stream",
|
||
Metadata={"infratest-run": self.run_id},
|
||
)
|
||
self.created_objects.add((bucket, key))
|
||
head = self.api_s3.head_object(Bucket=bucket, Key=key)
|
||
if head["ContentLength"] != len(self.payload):
|
||
raise AssertionError("HeadObject вернул неверный размер")
|
||
response = self.api_s3.get_object(Bucket=bucket, Key=key)
|
||
if response["Body"].read() != self.payload:
|
||
raise AssertionError("GetObject вернул другие байты")
|
||
listed = self.api_s3.list_objects_v2(
|
||
Bucket=bucket, Prefix=key, MaxKeys=2
|
||
).get("Contents", [])
|
||
if not any(item["Key"] == key for item in listed):
|
||
raise AssertionError("ListObjectsV2 не вернул тестовый объект")
|
||
return "Put/Head/Get/List с ВМ успешны"
|
||
|
||
def _check_presigned(self, key: str) -> str:
|
||
bucket = self.settings.buckets["quarantine"]
|
||
put_url = self.api_s3.generate_presigned_url(
|
||
"put_object",
|
||
Params={
|
||
"Bucket": bucket,
|
||
"Key": key,
|
||
"ContentType": "application/octet-stream",
|
||
},
|
||
ExpiresIn=300,
|
||
)
|
||
response = requests.put(
|
||
put_url,
|
||
data=self.payload,
|
||
headers={"Content-Type": "application/octet-stream"},
|
||
timeout=self.settings.timeout_seconds,
|
||
)
|
||
response.raise_for_status()
|
||
self.created_objects.add((bucket, key))
|
||
self.api_s3.head_object(Bucket=bucket, Key=key)
|
||
get_url = self.api_s3.generate_presigned_url(
|
||
"get_object",
|
||
Params={"Bucket": bucket, "Key": key},
|
||
ExpiresIn=300,
|
||
)
|
||
downloaded = requests.get(
|
||
get_url, timeout=self.settings.timeout_seconds
|
||
)
|
||
downloaded.raise_for_status()
|
||
if downloaded.content != self.payload:
|
||
raise AssertionError("presigned GET вернул другие байты")
|
||
return "presigned PUT/GET успешны, содержимое совпадает"
|
||
|
||
def _check_promote(self, source_key: str, destination_key: str) -> str:
|
||
quarantine = self.settings.buckets["quarantine"]
|
||
attachments = self.settings.buckets["attachments"]
|
||
self.api_s3.copy_object(
|
||
Bucket=attachments,
|
||
Key=destination_key,
|
||
CopySource={"Bucket": quarantine, "Key": source_key},
|
||
)
|
||
self.created_objects.add((attachments, destination_key))
|
||
copied = self.api_s3.get_object(
|
||
Bucket=attachments, Key=destination_key
|
||
)["Body"].read()
|
||
if copied != self.payload:
|
||
raise AssertionError("скопированный объект повреждён")
|
||
self.api_s3.delete_object(Bucket=quarantine, Key=source_key)
|
||
self.created_objects.discard((quarantine, source_key))
|
||
try:
|
||
self.api_s3.head_object(Bucket=quarantine, Key=source_key)
|
||
except ClientError as exc:
|
||
if exc.response.get("ResponseMetadata", {}).get("HTTPStatusCode") != 404:
|
||
raise
|
||
else:
|
||
raise AssertionError("исходный quarantine-объект не удалён")
|
||
return "CopyObject успешен, исходный объект удалён"
|
||
|
||
def _check_anonymous_denied(self, bucket: str, key: str) -> str:
|
||
signed_url = self.api_s3.generate_presigned_url(
|
||
"get_object",
|
||
Params={"Bucket": bucket, "Key": key},
|
||
ExpiresIn=300,
|
||
)
|
||
parsed = urlsplit(signed_url)
|
||
unsigned_url = urlunsplit(
|
||
(parsed.scheme, parsed.netloc, parsed.path, "", "")
|
||
)
|
||
response = requests.get(
|
||
unsigned_url,
|
||
allow_redirects=False,
|
||
timeout=self.settings.timeout_seconds,
|
||
)
|
||
if response.status_code not in {401, 403}:
|
||
raise AssertionError(
|
||
f"анонимный GET вернул HTTP {response.status_code}, ожидался 401/403"
|
||
)
|
||
return f"анонимный GET запрещён (HTTP {response.status_code})"
|
||
|
||
@staticmethod
|
||
def _assert_denied(operation: Callable[[], Any], description: str) -> None:
|
||
try:
|
||
operation()
|
||
except ClientError as exc:
|
||
status = exc.response.get("ResponseMetadata", {}).get("HTTPStatusCode")
|
||
code = exc.response.get("Error", {}).get("Code", "")
|
||
if status in {401, 403} or code in {
|
||
"AccessDenied",
|
||
"AllAccessDisabled",
|
||
"InvalidAccessKeyId",
|
||
}:
|
||
return
|
||
raise
|
||
raise AssertionError(f"операция неожиданно разрешена: {description}")
|
||
|
||
def _check_read_key_bucket_boundaries(self) -> str:
|
||
quarantine = self.settings.buckets["quarantine"]
|
||
self.read_s3.head_bucket(Bucket=quarantine)
|
||
self.read_s3.list_objects_v2(Bucket=quarantine, MaxKeys=1)
|
||
for logical_name in ("attachments", "documents"):
|
||
bucket = self.settings.buckets[logical_name]
|
||
self._assert_denied(
|
||
lambda bucket=bucket: self.read_s3.list_objects_v2(
|
||
Bucket=bucket, MaxKeys=1
|
||
),
|
||
f"read-key ListBucket {logical_name}",
|
||
)
|
||
return "read-key видит только quarantine"
|
||
|
||
def _check_read_key_object_permissions(
|
||
self,
|
||
quarantine_key: str,
|
||
attachments_key: str,
|
||
documents_key: str,
|
||
) -> str:
|
||
quarantine = self.settings.buckets["quarantine"]
|
||
attachments = self.settings.buckets["attachments"]
|
||
documents = self.settings.buckets["documents"]
|
||
response = self.read_s3.get_object(
|
||
Bucket=quarantine, Key=quarantine_key
|
||
)
|
||
if response["Body"].read() != self.payload:
|
||
raise AssertionError("read-key получил повреждённые данные")
|
||
|
||
denied_put_key = f"{self.prefix}/read-key-must-not-put.bin"
|
||
try:
|
||
self._assert_denied(
|
||
lambda: self.read_s3.put_object(
|
||
Bucket=quarantine, Key=denied_put_key, Body=self.payload
|
||
),
|
||
"read-key PutObject quarantine",
|
||
)
|
||
finally:
|
||
# Если политика ошибочно разрешила PUT, удалить объект API-ключом.
|
||
try:
|
||
self.api_s3.delete_object(
|
||
Bucket=quarantine, Key=denied_put_key
|
||
)
|
||
except Exception: # noqa: BLE001 - основная очистка будет продолжена
|
||
pass
|
||
|
||
self._assert_denied(
|
||
lambda: self.read_s3.delete_object(
|
||
Bucket=quarantine, Key=quarantine_key
|
||
),
|
||
"read-key DeleteObject quarantine",
|
||
)
|
||
self._assert_denied(
|
||
lambda: self.read_s3.get_object(
|
||
Bucket=attachments, Key=attachments_key
|
||
),
|
||
"read-key GetObject attachments",
|
||
)
|
||
self._assert_denied(
|
||
lambda: self.read_s3.get_object(
|
||
Bucket=documents, Key=documents_key
|
||
),
|
||
"read-key GetObject documents",
|
||
)
|
||
return "GET quarantine разрешён; запись, удаление и другие бакеты запрещены"
|
||
|
||
def _check_cors(self, key: str) -> str:
|
||
url = self.api_s3.generate_presigned_url(
|
||
"put_object",
|
||
Params={
|
||
"Bucket": self.settings.buckets["quarantine"],
|
||
"Key": key,
|
||
"ContentType": "application/octet-stream",
|
||
},
|
||
ExpiresIn=300,
|
||
)
|
||
response = requests.options(
|
||
url,
|
||
headers={
|
||
"Origin": self.settings.public_web_url or "",
|
||
"Access-Control-Request-Method": "PUT",
|
||
"Access-Control-Request-Headers": "content-type,x-amz-meta-infratest",
|
||
},
|
||
timeout=self.settings.timeout_seconds,
|
||
)
|
||
if response.status_code not in {200, 204}:
|
||
raise AssertionError(f"preflight вернул HTTP {response.status_code}")
|
||
allow_origin = response.headers.get("Access-Control-Allow-Origin")
|
||
if allow_origin != self.settings.public_web_url:
|
||
raise AssertionError(
|
||
f"Access-Control-Allow-Origin={allow_origin!r}, "
|
||
f"ожидался точный PUBLIC_WEB_URL"
|
||
)
|
||
allow_methods = response.headers.get("Access-Control-Allow-Methods", "")
|
||
if "PUT" not in allow_methods.upper():
|
||
raise AssertionError("CORS не разрешает PUT")
|
||
return "CORS разрешает presigned PUT только с PUBLIC_WEB_URL"
|
||
|
||
def _cleanup(self) -> None:
|
||
failures: list[str] = []
|
||
for bucket, key in sorted(self.created_objects):
|
||
try:
|
||
self.api_s3.delete_object(Bucket=bucket, Key=key)
|
||
except Exception as exc: # noqa: BLE001 - удалить остальные объекты
|
||
failures.append(f"{bucket}/{key}: {sanitize(exc)}")
|
||
if failures:
|
||
self.reporter.add(
|
||
"s3.cleanup",
|
||
"FAIL",
|
||
"не удалены тестовые объекты: " + "; ".join(failures),
|
||
)
|
||
else:
|
||
self.reporter.add(
|
||
"s3.cleanup",
|
||
"PASS",
|
||
f"удалены все объекты префикса {self.prefix}",
|
||
)
|
||
|
||
|
||
def parse_args(argv: list[str] | None = None) -> argparse.Namespace:
|
||
parser = argparse.ArgumentParser(
|
||
description="Проверка PostgreSQL и Selectel S3 для HAN Chat"
|
||
)
|
||
parser.add_argument(
|
||
"--env-file",
|
||
type=Path,
|
||
default=Path(__file__).with_name(".env"),
|
||
help="путь к .env (по умолчанию .env рядом со скриптом)",
|
||
)
|
||
parser.add_argument(
|
||
"--mode",
|
||
choices=("readonly", "full"),
|
||
default="full",
|
||
help="full выполняет самочищающиеся DDL/S3 операции",
|
||
)
|
||
parser.add_argument(
|
||
"--json-report",
|
||
type=Path,
|
||
help="сохранить обезличенный JSON-отчёт",
|
||
)
|
||
return parser.parse_args(argv)
|
||
|
||
|
||
def main(argv: list[str] | None = None) -> int:
|
||
args = parse_args(argv)
|
||
try:
|
||
settings = Settings.from_env_file(args.env_file.resolve())
|
||
except (ConfigError, ValueError) as exc:
|
||
print(f"Ошибка конфигурации: {sanitize(exc)}", file=sys.stderr)
|
||
return 2
|
||
|
||
reporter = Reporter()
|
||
test = InfraTest(settings, reporter, args.mode)
|
||
print(f"HAN Chat infrastructure test: mode={args.mode}, run={test.run_id[:8]}")
|
||
test.run()
|
||
if args.json_report:
|
||
reporter.write_json(args.json_report)
|
||
|
||
passed = sum(item.status == "PASS" for item in reporter.results)
|
||
failed = sum(item.status == "FAIL" for item in reporter.results)
|
||
skipped = sum(item.status == "SKIP" for item in reporter.results)
|
||
print(f"\nИтог: PASS={passed}, FAIL={failed}, SKIP={skipped}")
|
||
return reporter.exit_code
|
||
|
||
|
||
if __name__ == "__main__":
|
||
raise SystemExit(main())
|