Добавлен тест инфраструктуры
This commit is contained in:
mi
@@ -0,0 +1,844 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Самоочищающийся инфраструктурный тест PostgreSQL и S3 для HAN Chat."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
import uuid
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
from typing import Any, Callable
|
||||
from urllib.parse import urlsplit, urlunsplit
|
||||
|
||||
try:
|
||||
import boto3
|
||||
import psycopg
|
||||
import requests
|
||||
from botocore.config import Config as BotoConfig
|
||||
from botocore.exceptions import ClientError
|
||||
from dotenv import dotenv_values
|
||||
from psycopg import sql
|
||||
except ImportError as exc: # pragma: no cover - понятная ошибка до запуска тестов
|
||||
print(
|
||||
f"Не установлена зависимость {exc.name!r}. "
|
||||
"Выполните: python3 -m pip install -r requirements.txt",
|
||||
file=sys.stderr,
|
||||
)
|
||||
raise SystemExit(2) from exc
|
||||
|
||||
|
||||
URL_PATTERN = re.compile(r"https?://[^\s]+")
|
||||
EXPECTED_ROLES = (
|
||||
("han_app", "han_app", "HAN_PG_PASSWORD_HAN_APP"),
|
||||
("bitrix_local_app", "bitrix_local", "HAN_PG_PASSWORD_BITRIX"),
|
||||
("bitrix_sync_user", "bitrix_sync", "HAN_PG_PASSWORD_BITRIX_SYNC"),
|
||||
("message_safety_app", "message_safety", "HAN_PG_PASSWORD_MESSAGE_SAFETY"),
|
||||
("keycloak_user", "keycloak", "HAN_PG_PASSWORD_KEYCLOAK"),
|
||||
)
|
||||
EXPECTED_TABLES = {
|
||||
"han_app": (
|
||||
"alembic_version",
|
||||
"app_settings",
|
||||
"dialogs",
|
||||
"messages",
|
||||
"message_attachments",
|
||||
"documents",
|
||||
"sync_queue",
|
||||
),
|
||||
"bitrix_local": (
|
||||
"alembic_version",
|
||||
"portal_installations",
|
||||
"dialog_sessions",
|
||||
"inbox_events",
|
||||
"outbound_messages",
|
||||
),
|
||||
"bitrix_sync": ("alembic_version",),
|
||||
}
|
||||
|
||||
|
||||
class ConfigError(ValueError):
|
||||
"""Ошибка пользовательской конфигурации."""
|
||||
|
||||
|
||||
def as_bool(value: str | None, default: bool = False) -> bool:
|
||||
if value is None or not value.strip():
|
||||
return default
|
||||
normalized = value.strip().lower()
|
||||
if normalized in {"1", "true", "yes", "on"}:
|
||||
return True
|
||||
if normalized in {"0", "false", "no", "off"}:
|
||||
return False
|
||||
raise ConfigError(f"Ожидалось логическое значение, получено: {value!r}")
|
||||
|
||||
|
||||
def sanitize(value: Any) -> str:
|
||||
"""Удаляет секреты и query-параметры URL из диагностического текста."""
|
||||
text = str(value)
|
||||
text = URL_PATTERN.sub(
|
||||
lambda match: urlunsplit(
|
||||
(*urlsplit(match.group(0))[:3], "", "")
|
||||
),
|
||||
text,
|
||||
)
|
||||
text = re.sub(
|
||||
r"(?i)\b(password|secret|access[_-]?key|authorization|token|dsn)"
|
||||
r"\s*[:=]\s*[^\s,;]+",
|
||||
lambda match: f"{match.group(1)}=<redacted>",
|
||||
text,
|
||||
)
|
||||
return text[:1000]
|
||||
|
||||
|
||||
def require(values: dict[str, str], name: str) -> str:
|
||||
value = values.get(name, "").strip()
|
||||
if not value:
|
||||
raise ConfigError(f"Не задана обязательная переменная {name}")
|
||||
return value
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class PgRole:
|
||||
user: str
|
||||
schema: str
|
||||
password: str = field(repr=False)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Settings:
|
||||
pg_host: str
|
||||
pg_port: int
|
||||
pg_database: str
|
||||
pg_sslmode: str
|
||||
pg_sslrootcert: str
|
||||
pg_roles: tuple[PgRole, ...]
|
||||
s3_endpoint: str
|
||||
s3_region: str | None
|
||||
buckets: dict[str, str]
|
||||
s3_access_key: str = field(repr=False)
|
||||
s3_secret_key: str = field(repr=False)
|
||||
s3_read_access_key: str = field(repr=False)
|
||||
s3_read_secret_key: str = field(repr=False)
|
||||
public_web_url: str | None
|
||||
check_cors: bool
|
||||
check_migrations: bool
|
||||
timeout_seconds: int
|
||||
|
||||
@classmethod
|
||||
def from_env_file(cls, path: Path) -> "Settings":
|
||||
if not path.is_file():
|
||||
raise ConfigError(f"Файл настроек не найден: {path}")
|
||||
raw = dotenv_values(path)
|
||||
values = {
|
||||
key: str(value)
|
||||
for key, value in raw.items()
|
||||
if value is not None
|
||||
}
|
||||
# Явно экспортированные переменные имеют приоритет над файлом.
|
||||
values.update({key: value for key, value in os.environ.items() if key in values})
|
||||
|
||||
sslmode = values.get("HAN_PG_SSLMODE", "verify-full").strip()
|
||||
if sslmode != "verify-full":
|
||||
raise ConfigError("HAN_PG_SSLMODE должен быть verify-full")
|
||||
ca_path = require(values, "HAN_PG_SSLROOTCERT")
|
||||
if not Path(ca_path).is_file():
|
||||
raise ConfigError(f"CA-файл PostgreSQL не найден: {ca_path}")
|
||||
|
||||
endpoint = require(values, "SELECTEL_S3_ENDPOINT_URL")
|
||||
parsed_endpoint = urlsplit(endpoint)
|
||||
if parsed_endpoint.scheme != "https" or not parsed_endpoint.netloc:
|
||||
raise ConfigError("SELECTEL_S3_ENDPOINT_URL должен быть корректным HTTPS URL")
|
||||
|
||||
roles = tuple(
|
||||
PgRole(
|
||||
values.get(f"HAN_PG_USER_{env_suffix}", default_user).strip(),
|
||||
values.get(f"HAN_PG_SCHEMA_{env_suffix}", default_schema).strip(),
|
||||
require(values, password_env),
|
||||
)
|
||||
for default_user, default_schema, password_env in EXPECTED_ROLES
|
||||
for env_suffix in (
|
||||
{
|
||||
"han_app": "HAN_APP",
|
||||
"bitrix_local": "BITRIX",
|
||||
"bitrix_sync": "BITRIX_SYNC",
|
||||
"message_safety": "MESSAGE_SAFETY",
|
||||
"keycloak": "KEYCLOAK",
|
||||
}[default_schema],
|
||||
)
|
||||
)
|
||||
timeout = int(values.get("INFRATEST_TIMEOUT_SECONDS", "15"))
|
||||
if timeout < 1 or timeout > 300:
|
||||
raise ConfigError("INFRATEST_TIMEOUT_SECONDS должен быть от 1 до 300")
|
||||
|
||||
public_web_url = values.get("PUBLIC_WEB_URL", "").strip() or None
|
||||
check_cors = as_bool(values.get("INFRATEST_CHECK_CORS"), True)
|
||||
if check_cors and not public_web_url:
|
||||
raise ConfigError("Для CORS-проверки задайте PUBLIC_WEB_URL")
|
||||
|
||||
return cls(
|
||||
pg_host=require(values, "HAN_PG_HOST"),
|
||||
pg_port=int(values.get("HAN_PG_PORT", "5432")),
|
||||
pg_database=require(values, "HAN_PG_DATABASE"),
|
||||
pg_sslmode=sslmode,
|
||||
pg_sslrootcert=ca_path,
|
||||
pg_roles=roles,
|
||||
s3_endpoint=endpoint.rstrip("/"),
|
||||
s3_region=values.get("SELECTEL_S3_REGION", "").strip() or None,
|
||||
buckets={
|
||||
"quarantine": require(values, "SELECTEL_S3_BUCKET_QUARANTINE"),
|
||||
"attachments": require(values, "SELECTEL_S3_BUCKET_ATTACHMENTS"),
|
||||
"documents": require(values, "SELECTEL_S3_BUCKET_DOCUMENTS"),
|
||||
},
|
||||
s3_access_key=require(values, "SELECTEL_S3_ACCESS_KEY"),
|
||||
s3_secret_key=require(values, "SELECTEL_S3_SECRET_KEY"),
|
||||
s3_read_access_key=require(
|
||||
values, "SELECTEL_S3_QUARANTINE_READ_ACCESS_KEY"
|
||||
),
|
||||
s3_read_secret_key=require(
|
||||
values, "SELECTEL_S3_QUARANTINE_READ_SECRET_KEY"
|
||||
),
|
||||
public_web_url=public_web_url,
|
||||
check_cors=check_cors,
|
||||
check_migrations=as_bool(
|
||||
values.get("INFRATEST_CHECK_MIGRATIONS"), False
|
||||
),
|
||||
timeout_seconds=timeout,
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Result:
|
||||
check: str
|
||||
status: str
|
||||
detail: str
|
||||
|
||||
|
||||
class Reporter:
|
||||
def __init__(self) -> None:
|
||||
self.results: list[Result] = []
|
||||
|
||||
def add(self, check: str, status: str, detail: Any) -> None:
|
||||
result = Result(check, status, sanitize(detail))
|
||||
self.results.append(result)
|
||||
print(f"[{status:4}] {check}: {result.detail}")
|
||||
|
||||
def run(
|
||||
self,
|
||||
check: str,
|
||||
operation: Callable[[], Any],
|
||||
success_detail: str = "OK",
|
||||
) -> bool:
|
||||
try:
|
||||
detail = operation()
|
||||
except Exception as exc: # noqa: BLE001 - каждая проверка должна продолжить отчёт
|
||||
self.add(check, "FAIL", f"{type(exc).__name__}: {exc}")
|
||||
return False
|
||||
self.add(check, "PASS", success_detail if detail is None else detail)
|
||||
return True
|
||||
|
||||
def skip(self, check: str, detail: str) -> None:
|
||||
self.add(check, "SKIP", detail)
|
||||
|
||||
@property
|
||||
def exit_code(self) -> int:
|
||||
return 1 if any(item.status == "FAIL" for item in self.results) else 0
|
||||
|
||||
def write_json(self, path: Path) -> None:
|
||||
path.write_text(
|
||||
json.dumps(
|
||||
[item.__dict__ for item in self.results],
|
||||
ensure_ascii=False,
|
||||
indent=2,
|
||||
)
|
||||
+ "\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
|
||||
class InfraTest:
|
||||
def __init__(self, settings: Settings, reporter: Reporter, mode: str) -> None:
|
||||
self.settings = settings
|
||||
self.reporter = reporter
|
||||
self.mode = mode
|
||||
self.run_id = uuid.uuid4().hex
|
||||
self.prefix = f"infratest/{self.run_id}"
|
||||
self.payload = (
|
||||
b"HAN Chat infrastructure test\n"
|
||||
+ self.run_id.encode("ascii")
|
||||
+ b"\n"
|
||||
)
|
||||
self.api_s3 = self._s3_client(
|
||||
settings.s3_access_key, settings.s3_secret_key
|
||||
)
|
||||
self.read_s3 = self._s3_client(
|
||||
settings.s3_read_access_key, settings.s3_read_secret_key
|
||||
)
|
||||
self.created_objects: set[tuple[str, str]] = set()
|
||||
|
||||
def _s3_client(self, access_key: str, secret_key: str):
|
||||
kwargs: dict[str, Any] = {
|
||||
"service_name": "s3",
|
||||
"endpoint_url": self.settings.s3_endpoint,
|
||||
"aws_access_key_id": access_key,
|
||||
"aws_secret_access_key": secret_key,
|
||||
"config": BotoConfig(
|
||||
connect_timeout=self.settings.timeout_seconds,
|
||||
read_timeout=self.settings.timeout_seconds,
|
||||
retries={"max_attempts": 2},
|
||||
),
|
||||
}
|
||||
if self.settings.s3_region:
|
||||
kwargs["region_name"] = self.settings.s3_region
|
||||
return boto3.client(**kwargs)
|
||||
|
||||
def _pg_connect(self, role: PgRole):
|
||||
return psycopg.connect(
|
||||
host=self.settings.pg_host,
|
||||
port=self.settings.pg_port,
|
||||
dbname=self.settings.pg_database,
|
||||
user=role.user,
|
||||
password=role.password,
|
||||
sslmode=self.settings.pg_sslmode,
|
||||
sslrootcert=self.settings.pg_sslrootcert,
|
||||
connect_timeout=self.settings.timeout_seconds,
|
||||
application_name=f"han-infratest-{self.run_id[:8]}",
|
||||
)
|
||||
|
||||
def run(self) -> None:
|
||||
self.run_postgres()
|
||||
self.run_s3()
|
||||
|
||||
def run_postgres(self) -> None:
|
||||
print("\nPostgreSQL")
|
||||
schemas = tuple(role.schema for role in self.settings.pg_roles)
|
||||
for role in self.settings.pg_roles:
|
||||
connection_ok = self.reporter.run(
|
||||
f"postgres.{role.user}.connect_tls",
|
||||
lambda role=role: self._check_pg_connection(role),
|
||||
)
|
||||
if not connection_ok:
|
||||
continue
|
||||
self.reporter.run(
|
||||
f"postgres.{role.user}.schema_isolation",
|
||||
lambda role=role: self._check_schema_isolation(role, schemas),
|
||||
)
|
||||
if self.mode == "full":
|
||||
self.reporter.run(
|
||||
f"postgres.{role.user}.ddl_dml_rollback",
|
||||
lambda role=role: self._check_pg_write(role),
|
||||
)
|
||||
else:
|
||||
self.reporter.skip(
|
||||
f"postgres.{role.user}.ddl_dml_rollback",
|
||||
"режим readonly",
|
||||
)
|
||||
|
||||
self.reporter.run("postgres.pgcrypto", self._check_pgcrypto)
|
||||
if self.settings.check_migrations:
|
||||
self.reporter.run("postgres.migrations", self._check_migrations)
|
||||
else:
|
||||
self.reporter.skip(
|
||||
"postgres.migrations",
|
||||
"INFRATEST_CHECK_MIGRATIONS=false",
|
||||
)
|
||||
|
||||
def _check_pg_connection(self, role: PgRole) -> str:
|
||||
with self._pg_connect(role) as conn:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute(
|
||||
"""
|
||||
SELECT current_user, current_setting('search_path'),
|
||||
EXISTS (
|
||||
SELECT 1 FROM pg_stat_ssl
|
||||
WHERE pid = pg_backend_pid() AND ssl
|
||||
)
|
||||
"""
|
||||
)
|
||||
current_user, search_path, tls_active = cur.fetchone()
|
||||
if current_user != role.user:
|
||||
raise AssertionError(
|
||||
f"current_user={current_user!r}, ожидался {role.user!r}"
|
||||
)
|
||||
if role.schema not in search_path.split(","):
|
||||
raise AssertionError(
|
||||
f"search_path={search_path!r}, ожидалась {role.schema!r}"
|
||||
)
|
||||
if not tls_active:
|
||||
raise AssertionError("соединение установлено без TLS")
|
||||
return f"TLS active, search_path={search_path}"
|
||||
|
||||
def _check_schema_isolation(
|
||||
self, role: PgRole, all_schemas: tuple[str, ...]
|
||||
) -> str:
|
||||
with self._pg_connect(role) as conn:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute(
|
||||
"""
|
||||
SELECT EXISTS (
|
||||
SELECT 1 FROM information_schema.schemata
|
||||
WHERE schema_name = %s
|
||||
), has_schema_privilege(current_user, %s, 'USAGE')
|
||||
""",
|
||||
(role.schema, role.schema),
|
||||
)
|
||||
schema_exists, own_usage = cur.fetchone()
|
||||
if not schema_exists or not own_usage:
|
||||
raise AssertionError(
|
||||
f"нет схемы или USAGE для {role.schema}"
|
||||
)
|
||||
unexpected = []
|
||||
for other in all_schemas:
|
||||
if other == role.schema:
|
||||
continue
|
||||
cur.execute(
|
||||
"SELECT has_schema_privilege(current_user, %s, 'USAGE')",
|
||||
(other,),
|
||||
)
|
||||
if cur.fetchone()[0]:
|
||||
unexpected.append(other)
|
||||
if unexpected:
|
||||
raise AssertionError(
|
||||
"обнаружен USAGE на чужие схемы: " + ", ".join(unexpected)
|
||||
)
|
||||
return f"доступ только к схеме {role.schema}"
|
||||
|
||||
def _check_pg_write(self, role: PgRole) -> str:
|
||||
table_name = f"_infratest_{self.run_id}"
|
||||
with self._pg_connect(role) as conn:
|
||||
try:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute(
|
||||
sql.SQL("CREATE TABLE {}.{} (id integer PRIMARY KEY, value text)")
|
||||
.format(sql.Identifier(role.schema), sql.Identifier(table_name))
|
||||
)
|
||||
cur.execute(
|
||||
sql.SQL("INSERT INTO {}.{} VALUES (%s, %s)").format(
|
||||
sql.Identifier(role.schema), sql.Identifier(table_name)
|
||||
),
|
||||
(1, "ok"),
|
||||
)
|
||||
cur.execute(
|
||||
sql.SQL("SELECT value FROM {}.{} WHERE id = %s").format(
|
||||
sql.Identifier(role.schema), sql.Identifier(table_name)
|
||||
),
|
||||
(1,),
|
||||
)
|
||||
if cur.fetchone()[0] != "ok":
|
||||
raise AssertionError("прочитано неожиданное значение")
|
||||
finally:
|
||||
conn.rollback()
|
||||
with conn.cursor() as cur:
|
||||
cur.execute("SELECT to_regclass(%s)", (f"{role.schema}.{table_name}",))
|
||||
if cur.fetchone()[0] is not None:
|
||||
raise AssertionError("таблица сохранилась после ROLLBACK")
|
||||
return "CREATE/INSERT/SELECT успешны, DDL откачен"
|
||||
|
||||
def _check_pgcrypto(self) -> str:
|
||||
role = next(item for item in self.settings.pg_roles if item.schema == "han_app")
|
||||
with self._pg_connect(role) as conn:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute(
|
||||
"SELECT EXISTS (SELECT 1 FROM pg_extension WHERE extname='pgcrypto')"
|
||||
)
|
||||
if not cur.fetchone()[0]:
|
||||
raise AssertionError("расширение pgcrypto не установлено")
|
||||
return "расширение pgcrypto установлено"
|
||||
|
||||
def _check_migrations(self) -> str:
|
||||
role_by_schema = {role.schema: role for role in self.settings.pg_roles}
|
||||
missing: list[str] = []
|
||||
revisions: dict[str, str] = {}
|
||||
for schema, tables in EXPECTED_TABLES.items():
|
||||
with self._pg_connect(role_by_schema[schema]) as conn:
|
||||
with conn.cursor() as cur:
|
||||
for table in tables:
|
||||
cur.execute("SELECT to_regclass(%s)", (f"{schema}.{table}",))
|
||||
if cur.fetchone()[0] is None:
|
||||
missing.append(f"{schema}.{table}")
|
||||
cur.execute(
|
||||
sql.SQL("SELECT version_num FROM {}.alembic_version LIMIT 1")
|
||||
.format(sql.Identifier(schema))
|
||||
)
|
||||
row = cur.fetchone()
|
||||
revisions[schema] = row[0] if row else "<empty>"
|
||||
if missing:
|
||||
raise AssertionError("нет таблиц: " + ", ".join(missing))
|
||||
if revisions.get("han_app") != "0001_initial":
|
||||
raise AssertionError(
|
||||
f"han_app revision={revisions.get('han_app')!r}, ожидался '0001_initial'"
|
||||
)
|
||||
return "таблицы и Alembic revisions присутствуют"
|
||||
|
||||
def run_s3(self) -> None:
|
||||
print("\nSelectel S3")
|
||||
reachable: dict[str, bool] = {}
|
||||
for logical_name, bucket in self.settings.buckets.items():
|
||||
reachable[logical_name] = self.reporter.run(
|
||||
f"s3.{logical_name}.head_bucket",
|
||||
lambda bucket=bucket: self.api_s3.head_bucket(Bucket=bucket),
|
||||
f"бакет {bucket} доступен",
|
||||
)
|
||||
|
||||
self.reporter.run(
|
||||
"s3.quarantine_read_key.boundaries",
|
||||
self._check_read_key_bucket_boundaries,
|
||||
)
|
||||
if self.mode != "full":
|
||||
for name in (
|
||||
"direct_vm_operations",
|
||||
"presigned_put_get",
|
||||
"copy_promote",
|
||||
"cors",
|
||||
"anonymous_access",
|
||||
"quarantine_read_key.object_permissions",
|
||||
):
|
||||
self.reporter.skip(f"s3.{name}", "режим readonly")
|
||||
return
|
||||
if not all(reachable.values()):
|
||||
self.reporter.add(
|
||||
"s3.full_operations",
|
||||
"FAIL",
|
||||
"полные проверки невозможны: не все бакеты доступны",
|
||||
)
|
||||
return
|
||||
|
||||
try:
|
||||
direct_keys: dict[str, str] = {}
|
||||
for logical_name, bucket in self.settings.buckets.items():
|
||||
key = f"{self.prefix}/direct-{logical_name}.bin"
|
||||
direct_keys[logical_name] = key
|
||||
self.reporter.run(
|
||||
f"s3.{logical_name}.direct_vm_operations",
|
||||
lambda bucket=bucket, key=key: self._check_direct_object(
|
||||
bucket, key
|
||||
),
|
||||
)
|
||||
|
||||
presigned_key = f"{self.prefix}/presigned.bin"
|
||||
presigned_ok = self.reporter.run(
|
||||
"s3.quarantine.presigned_put_get",
|
||||
lambda: self._check_presigned(presigned_key),
|
||||
)
|
||||
if presigned_ok:
|
||||
promoted_key = f"{self.prefix}/promoted.bin"
|
||||
self.reporter.run(
|
||||
"s3.quarantine_to_attachments.copy_promote",
|
||||
lambda: self._check_promote(presigned_key, promoted_key),
|
||||
)
|
||||
|
||||
privacy_key = direct_keys["documents"]
|
||||
self.reporter.run(
|
||||
"s3.documents.anonymous_access_denied",
|
||||
lambda: self._check_anonymous_denied(
|
||||
self.settings.buckets["documents"], privacy_key
|
||||
),
|
||||
)
|
||||
self.reporter.run(
|
||||
"s3.quarantine_read_key.object_permissions",
|
||||
lambda: self._check_read_key_object_permissions(
|
||||
direct_keys["quarantine"],
|
||||
direct_keys["attachments"],
|
||||
direct_keys["documents"],
|
||||
),
|
||||
)
|
||||
if self.settings.check_cors:
|
||||
cors_key = f"{self.prefix}/cors.bin"
|
||||
self.reporter.run(
|
||||
"s3.quarantine.cors_preflight",
|
||||
lambda: self._check_cors(cors_key),
|
||||
)
|
||||
else:
|
||||
self.reporter.skip(
|
||||
"s3.quarantine.cors_preflight",
|
||||
"INFRATEST_CHECK_CORS=false",
|
||||
)
|
||||
finally:
|
||||
self._cleanup()
|
||||
|
||||
def _check_direct_object(self, bucket: str, key: str) -> str:
|
||||
self.api_s3.put_object(
|
||||
Bucket=bucket,
|
||||
Key=key,
|
||||
Body=self.payload,
|
||||
ContentType="application/octet-stream",
|
||||
Metadata={"infratest-run": self.run_id},
|
||||
)
|
||||
self.created_objects.add((bucket, key))
|
||||
head = self.api_s3.head_object(Bucket=bucket, Key=key)
|
||||
if head["ContentLength"] != len(self.payload):
|
||||
raise AssertionError("HeadObject вернул неверный размер")
|
||||
response = self.api_s3.get_object(Bucket=bucket, Key=key)
|
||||
if response["Body"].read() != self.payload:
|
||||
raise AssertionError("GetObject вернул другие байты")
|
||||
listed = self.api_s3.list_objects_v2(
|
||||
Bucket=bucket, Prefix=key, MaxKeys=2
|
||||
).get("Contents", [])
|
||||
if not any(item["Key"] == key for item in listed):
|
||||
raise AssertionError("ListObjectsV2 не вернул тестовый объект")
|
||||
return "Put/Head/Get/List с ВМ успешны"
|
||||
|
||||
def _check_presigned(self, key: str) -> str:
|
||||
bucket = self.settings.buckets["quarantine"]
|
||||
put_url = self.api_s3.generate_presigned_url(
|
||||
"put_object",
|
||||
Params={
|
||||
"Bucket": bucket,
|
||||
"Key": key,
|
||||
"ContentType": "application/octet-stream",
|
||||
},
|
||||
ExpiresIn=300,
|
||||
)
|
||||
response = requests.put(
|
||||
put_url,
|
||||
data=self.payload,
|
||||
headers={"Content-Type": "application/octet-stream"},
|
||||
timeout=self.settings.timeout_seconds,
|
||||
)
|
||||
response.raise_for_status()
|
||||
self.created_objects.add((bucket, key))
|
||||
self.api_s3.head_object(Bucket=bucket, Key=key)
|
||||
get_url = self.api_s3.generate_presigned_url(
|
||||
"get_object",
|
||||
Params={"Bucket": bucket, "Key": key},
|
||||
ExpiresIn=300,
|
||||
)
|
||||
downloaded = requests.get(
|
||||
get_url, timeout=self.settings.timeout_seconds
|
||||
)
|
||||
downloaded.raise_for_status()
|
||||
if downloaded.content != self.payload:
|
||||
raise AssertionError("presigned GET вернул другие байты")
|
||||
return "presigned PUT/GET успешны, содержимое совпадает"
|
||||
|
||||
def _check_promote(self, source_key: str, destination_key: str) -> str:
|
||||
quarantine = self.settings.buckets["quarantine"]
|
||||
attachments = self.settings.buckets["attachments"]
|
||||
self.api_s3.copy_object(
|
||||
Bucket=attachments,
|
||||
Key=destination_key,
|
||||
CopySource={"Bucket": quarantine, "Key": source_key},
|
||||
)
|
||||
self.created_objects.add((attachments, destination_key))
|
||||
copied = self.api_s3.get_object(
|
||||
Bucket=attachments, Key=destination_key
|
||||
)["Body"].read()
|
||||
if copied != self.payload:
|
||||
raise AssertionError("скопированный объект повреждён")
|
||||
self.api_s3.delete_object(Bucket=quarantine, Key=source_key)
|
||||
self.created_objects.discard((quarantine, source_key))
|
||||
try:
|
||||
self.api_s3.head_object(Bucket=quarantine, Key=source_key)
|
||||
except ClientError as exc:
|
||||
if exc.response.get("ResponseMetadata", {}).get("HTTPStatusCode") != 404:
|
||||
raise
|
||||
else:
|
||||
raise AssertionError("исходный quarantine-объект не удалён")
|
||||
return "CopyObject успешен, исходный объект удалён"
|
||||
|
||||
def _check_anonymous_denied(self, bucket: str, key: str) -> str:
|
||||
signed_url = self.api_s3.generate_presigned_url(
|
||||
"get_object",
|
||||
Params={"Bucket": bucket, "Key": key},
|
||||
ExpiresIn=300,
|
||||
)
|
||||
parsed = urlsplit(signed_url)
|
||||
unsigned_url = urlunsplit(
|
||||
(parsed.scheme, parsed.netloc, parsed.path, "", "")
|
||||
)
|
||||
response = requests.get(
|
||||
unsigned_url,
|
||||
allow_redirects=False,
|
||||
timeout=self.settings.timeout_seconds,
|
||||
)
|
||||
if response.status_code not in {401, 403}:
|
||||
raise AssertionError(
|
||||
f"анонимный GET вернул HTTP {response.status_code}, ожидался 401/403"
|
||||
)
|
||||
return f"анонимный GET запрещён (HTTP {response.status_code})"
|
||||
|
||||
@staticmethod
|
||||
def _assert_denied(operation: Callable[[], Any], description: str) -> None:
|
||||
try:
|
||||
operation()
|
||||
except ClientError as exc:
|
||||
status = exc.response.get("ResponseMetadata", {}).get("HTTPStatusCode")
|
||||
code = exc.response.get("Error", {}).get("Code", "")
|
||||
if status in {401, 403} or code in {
|
||||
"AccessDenied",
|
||||
"AllAccessDisabled",
|
||||
"InvalidAccessKeyId",
|
||||
}:
|
||||
return
|
||||
raise
|
||||
raise AssertionError(f"операция неожиданно разрешена: {description}")
|
||||
|
||||
def _check_read_key_bucket_boundaries(self) -> str:
|
||||
quarantine = self.settings.buckets["quarantine"]
|
||||
self.read_s3.head_bucket(Bucket=quarantine)
|
||||
self.read_s3.list_objects_v2(Bucket=quarantine, MaxKeys=1)
|
||||
for logical_name in ("attachments", "documents"):
|
||||
bucket = self.settings.buckets[logical_name]
|
||||
self._assert_denied(
|
||||
lambda bucket=bucket: self.read_s3.list_objects_v2(
|
||||
Bucket=bucket, MaxKeys=1
|
||||
),
|
||||
f"read-key ListBucket {logical_name}",
|
||||
)
|
||||
return "read-key видит только quarantine"
|
||||
|
||||
def _check_read_key_object_permissions(
|
||||
self,
|
||||
quarantine_key: str,
|
||||
attachments_key: str,
|
||||
documents_key: str,
|
||||
) -> str:
|
||||
quarantine = self.settings.buckets["quarantine"]
|
||||
attachments = self.settings.buckets["attachments"]
|
||||
documents = self.settings.buckets["documents"]
|
||||
response = self.read_s3.get_object(
|
||||
Bucket=quarantine, Key=quarantine_key
|
||||
)
|
||||
if response["Body"].read() != self.payload:
|
||||
raise AssertionError("read-key получил повреждённые данные")
|
||||
|
||||
denied_put_key = f"{self.prefix}/read-key-must-not-put.bin"
|
||||
try:
|
||||
self._assert_denied(
|
||||
lambda: self.read_s3.put_object(
|
||||
Bucket=quarantine, Key=denied_put_key, Body=self.payload
|
||||
),
|
||||
"read-key PutObject quarantine",
|
||||
)
|
||||
finally:
|
||||
# Если политика ошибочно разрешила PUT, удалить объект API-ключом.
|
||||
try:
|
||||
self.api_s3.delete_object(
|
||||
Bucket=quarantine, Key=denied_put_key
|
||||
)
|
||||
except Exception: # noqa: BLE001 - основная очистка будет продолжена
|
||||
pass
|
||||
|
||||
self._assert_denied(
|
||||
lambda: self.read_s3.delete_object(
|
||||
Bucket=quarantine, Key=quarantine_key
|
||||
),
|
||||
"read-key DeleteObject quarantine",
|
||||
)
|
||||
self._assert_denied(
|
||||
lambda: self.read_s3.get_object(
|
||||
Bucket=attachments, Key=attachments_key
|
||||
),
|
||||
"read-key GetObject attachments",
|
||||
)
|
||||
self._assert_denied(
|
||||
lambda: self.read_s3.get_object(
|
||||
Bucket=documents, Key=documents_key
|
||||
),
|
||||
"read-key GetObject documents",
|
||||
)
|
||||
return "GET quarantine разрешён; запись, удаление и другие бакеты запрещены"
|
||||
|
||||
def _check_cors(self, key: str) -> str:
|
||||
url = self.api_s3.generate_presigned_url(
|
||||
"put_object",
|
||||
Params={
|
||||
"Bucket": self.settings.buckets["quarantine"],
|
||||
"Key": key,
|
||||
"ContentType": "application/octet-stream",
|
||||
},
|
||||
ExpiresIn=300,
|
||||
)
|
||||
response = requests.options(
|
||||
url,
|
||||
headers={
|
||||
"Origin": self.settings.public_web_url or "",
|
||||
"Access-Control-Request-Method": "PUT",
|
||||
"Access-Control-Request-Headers": "content-type,x-amz-meta-infratest",
|
||||
},
|
||||
timeout=self.settings.timeout_seconds,
|
||||
)
|
||||
if response.status_code not in {200, 204}:
|
||||
raise AssertionError(f"preflight вернул HTTP {response.status_code}")
|
||||
allow_origin = response.headers.get("Access-Control-Allow-Origin")
|
||||
if allow_origin != self.settings.public_web_url:
|
||||
raise AssertionError(
|
||||
f"Access-Control-Allow-Origin={allow_origin!r}, "
|
||||
f"ожидался точный PUBLIC_WEB_URL"
|
||||
)
|
||||
allow_methods = response.headers.get("Access-Control-Allow-Methods", "")
|
||||
if "PUT" not in allow_methods.upper():
|
||||
raise AssertionError("CORS не разрешает PUT")
|
||||
return "CORS разрешает presigned PUT только с PUBLIC_WEB_URL"
|
||||
|
||||
def _cleanup(self) -> None:
|
||||
failures: list[str] = []
|
||||
for bucket, key in sorted(self.created_objects):
|
||||
try:
|
||||
self.api_s3.delete_object(Bucket=bucket, Key=key)
|
||||
except Exception as exc: # noqa: BLE001 - удалить остальные объекты
|
||||
failures.append(f"{bucket}/{key}: {sanitize(exc)}")
|
||||
if failures:
|
||||
self.reporter.add(
|
||||
"s3.cleanup",
|
||||
"FAIL",
|
||||
"не удалены тестовые объекты: " + "; ".join(failures),
|
||||
)
|
||||
else:
|
||||
self.reporter.add(
|
||||
"s3.cleanup",
|
||||
"PASS",
|
||||
f"удалены все объекты префикса {self.prefix}",
|
||||
)
|
||||
|
||||
|
||||
def parse_args(argv: list[str] | None = None) -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Проверка PostgreSQL и Selectel S3 для HAN Chat"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--env-file",
|
||||
type=Path,
|
||||
default=Path(__file__).with_name(".env"),
|
||||
help="путь к .env (по умолчанию .env рядом со скриптом)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--mode",
|
||||
choices=("readonly", "full"),
|
||||
default="full",
|
||||
help="full выполняет самочищающиеся DDL/S3 операции",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--json-report",
|
||||
type=Path,
|
||||
help="сохранить обезличенный JSON-отчёт",
|
||||
)
|
||||
return parser.parse_args(argv)
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
args = parse_args(argv)
|
||||
try:
|
||||
settings = Settings.from_env_file(args.env_file.resolve())
|
||||
except (ConfigError, ValueError) as exc:
|
||||
print(f"Ошибка конфигурации: {sanitize(exc)}", file=sys.stderr)
|
||||
return 2
|
||||
|
||||
reporter = Reporter()
|
||||
test = InfraTest(settings, reporter, args.mode)
|
||||
print(f"HAN Chat infrastructure test: mode={args.mode}, run={test.run_id[:8]}")
|
||||
test.run()
|
||||
if args.json_report:
|
||||
reporter.write_json(args.json_report)
|
||||
|
||||
passed = sum(item.status == "PASS" for item in reporter.results)
|
||||
failed = sum(item.status == "FAIL" for item in reporter.results)
|
||||
skipped = sum(item.status == "SKIP" for item in reporter.results)
|
||||
print(f"\nИтог: PASS={passed}, FAIL={failed}, SKIP={skipped}")
|
||||
return reporter.exit_code
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
Reference in New Issue
Block a user