Реализованы сервисы ВМ2 - проверка сообщений и синхронизация с Б24 (деплой еще без перевода в боевой режим)

2026-08-13 18:52:42 +03:00
parent 5100ba9fc3
commit 99605b1c77
144 changed files with 15295 additions and 1120 deletions
@@ -0,0 +1,316 @@
+#!/usr/bin/env python3
+"""Fail-closed VM2 adaptation of the reviewed HAN Selectel secrets loader."""
+
+from __future__ import annotations
+
+import argparse
+import base64
+import binascii
+import json
+import os
+import random
+import re
+import ssl
+import stat
+import sys
+import tempfile
+import time
+import urllib.error
+import urllib.parse
+import urllib.request
+from pathlib import Path
+from typing import Any, Mapping
+
+IDENTITY_URL = "https://cloud.api.selcloud.ru/identity/v3/auth/tokens"
+NAME_RE = re.compile(r"^[A-Z][A-Z0-9_]*$")
+CONSUMER_RE = re.compile(r"^[a-zA-Z0-9][a-zA-Z0-9_.-]*$")
+RETRYABLE = {408, 425, 429, 500, 502, 503, 504}
+MAX_CONFIG = 1_048_576
+
+
+class LoaderError(Exception):
+    """Expected error whose text contains no provider response or secret value."""
+
+
+def fail(message: str) -> None:
+    raise LoaderError(message)
+
+
+def private_file(path: Path, label: str) -> None:
+    try:
+        metadata = path.lstat()
+    except OSError as exc:
+        fail(f"cannot inspect {label}: {exc.__class__.__name__}")
+    if stat.S_ISLNK(metadata.st_mode) or not stat.S_ISREG(metadata.st_mode):
+        fail(f"{label} must be a regular non-symlink file")
+    if os.name != "nt" and stat.S_IMODE(metadata.st_mode) & 0o077:
+        fail(f"{label} must not be accessible by group or other users")
+
+
+def read_limited(path: Path, limit: int, label: str) -> bytes:
+    try:
+        with path.open("rb") as stream:
+            value = stream.read(limit + 1)
+    except OSError as exc:
+        fail(f"cannot read {label}: {exc.__class__.__name__}")
+    if len(value) > limit:
+        fail(f"{label} exceeds configured limit")
+    return value
+
+
+def object_value(value: Any, label: str) -> dict[str, Any]:
+    if not isinstance(value, dict):
+        fail(f"{label} must be an object")
+    return value
+
+
+def load_json(path: Path) -> dict[str, Any]:
+    try:
+        return object_value(json.loads(read_limited(path, MAX_CONFIG, "configuration")), "configuration")
+    except (UnicodeDecodeError, json.JSONDecodeError):
+        fail("configuration is not valid UTF-8 JSON")
+
+
+def required_string(value: Mapping[str, Any], key: str, label: str) -> str:
+    result = value.get(key)
+    if not isinstance(result, str) or not result:
+        fail(f"{label}.{key} must be a non-empty string")
+    return result
+
+
+class NoRedirect(urllib.request.HTTPRedirectHandler):
+    def redirect_request(self, req: Any, fp: Any, code: int, msg: str, headers: Any, newurl: str) -> None:
+        return None
+
+
+class Client:
+    def __init__(self, timeout: float, retries: int, maximum: int, ca_file: str | None) -> None:
+        context = ssl.create_default_context(cafile=ca_file)
+        self.opener = urllib.request.build_opener(
+            urllib.request.HTTPSHandler(context=context), NoRedirect()
+        )
+        self.timeout, self.retries, self.maximum = timeout, retries, maximum
+
+    def request(
+        self, method: str, url: str, expected: set[int], headers: Mapping[str, str] | None = None, body: bytes | None = None
+    ) -> tuple[Mapping[str, str], bytes]:
+        parsed = urllib.parse.urlsplit(url)
+        if parsed.scheme != "https" or not parsed.netloc or parsed.username or parsed.password:
+            fail("provider endpoint must be credential-free HTTPS")
+        request = urllib.request.Request(url, data=body, headers=dict(headers or {}), method=method)
+        for attempt in range(self.retries + 1):
+            try:
+                with self.opener.open(request, timeout=self.timeout) as response:
+                    if int(response.headers.get("Content-Length", 0)) > self.maximum:
+                        fail("provider response exceeds configured limit")
+                    response_body = response.read(self.maximum + 1)
+                    if len(response_body) > self.maximum:
+                        fail("provider response exceeds configured limit")
+                    if response.status not in expected:
+                        fail(f"provider request failed with HTTP {response.status}")
+                    return response.headers, response_body
+            except urllib.error.HTTPError as exc:
+                if exc.code not in RETRYABLE or attempt == self.retries:
+                    fail(f"provider request failed with HTTP {exc.code}")
+            except (urllib.error.URLError, TimeoutError, OSError):
+                if attempt == self.retries:
+                    fail("provider request failed after retries")
+            time.sleep(min(8.0, 0.25 * (2**attempt)) * (0.5 + random.random()))
+        fail("provider request failed")
+
+
+def credential(selectel: Mapping[str, Any], environ: Mapping[str, str]) -> str:
+    configured = Path(required_string(selectel, "password_file", "selectel"))
+    if not configured.is_absolute():
+        directory = environ.get("CREDENTIALS_DIRECTORY")
+        if not directory:
+            fail("relative password_file requires CREDENTIALS_DIRECTORY")
+        configured = Path(directory) / configured
+    private_file(configured, "Selectel credential")
+    try:
+        value = read_limited(configured, 16_384, "Selectel credential").decode().rstrip("\r\n")
+    except UnicodeDecodeError:
+        fail("Selectel credential is not UTF-8")
+    if not value or "\n" in value or "\r" in value:
+        fail("Selectel credential must contain one non-empty line")
+    return value
+
+
+def decode_document(raw: bytes, label: str) -> dict[str, Any]:
+    try:
+        return object_value(json.loads(raw.decode()), label)
+    except (UnicodeDecodeError, json.JSONDecodeError):
+        fail(f"{label} is not valid JSON")
+
+
+def fetch_values(config: Mapping[str, Any], specs: Mapping[str, Mapping[str, Any]], environ: Mapping[str, str]) -> dict[str, bytes]:
+    selectel = object_value(config.get("selectel"), "selectel")
+    http = object_value(config.get("http", {}), "http")
+    client = Client(
+        float(http.get("timeout_seconds", 10)),
+        int(http.get("retries", 3)),
+        int(http.get("max_response_bytes", MAX_CONFIG)),
+        selectel.get("ca_file"),
+    )
+    account = required_string(selectel, "account_id", "selectel")
+    auth = {
+        "auth": {
+            "identity": {"methods": ["password"], "password": {"user": {
+                "name": required_string(selectel, "username", "selectel"),
+                "domain": {"name": account},
+                "password": credential(selectel, environ),
+            }}},
+            "scope": {"project": {
+                "name": required_string(selectel, "project_name", "selectel"),
+                "domain": {"name": account},
+            }},
+        }
+    }
+    headers, body = client.request(
+        "POST",
+        str(selectel.get("identity_url", IDENTITY_URL)),
+        {201},
+        {"Content-Type": "application/json", "Accept": "application/json"},
+        json.dumps(auth, separators=(",", ":")).encode(),
+    )
+    token = headers.get("X-Subject-Token")
+    identity = decode_document(body, "identity response").get("token")
+    if not token or not isinstance(identity, dict) or not isinstance(identity.get("project"), dict):
+        fail("identity token is missing or not project-scoped")
+    matches: list[str] = []
+    for service in identity.get("catalog", []):
+        if isinstance(service, dict) and service.get("type") == "secrets-manager":
+            for endpoint in service.get("endpoints", []):
+                if (
+                    isinstance(endpoint, dict)
+                    and endpoint.get("region") == selectel.get("region")
+                    and endpoint.get("interface") == selectel.get("interface", "public")
+                    and isinstance(endpoint.get("url"), str)
+                ):
+                    matches.append(endpoint["url"].rstrip("/"))
+    if len(matches) != 1:
+        fail("service catalog has no unique matching Secrets Manager endpoint")
+    values: dict[str, bytes] = {}
+    for name, spec in specs.items():
+        remote = required_string(spec, "remote", f"secrets.{name}")
+        _, secret_body = client.request(
+            "GET",
+            matches[0] + "/v1/" + urllib.parse.quote(remote, safe=""),
+            {200},
+            {"X-Auth-Token": str(token), "Accept": "application/json"},
+        )
+        document = decode_document(secret_body, f"secret {name} response")
+        payload = document.get("version") if isinstance(document.get("version"), dict) else document
+        encoded = payload.get("value")
+        try:
+            value = base64.b64decode(encoded, validate=True)
+        except (TypeError, ValueError, binascii.Error):
+            fail(f"secret {name} has invalid encoding")
+        maximum = int(spec.get("max_bytes", 65_536))
+        if not value or len(value) > maximum or b"\x00" in value:
+            fail(f"secret {name} is empty, unsafe, or exceeds its limit")
+        values[name] = value
+    return values
+
+
+def file_values(config: Mapping[str, Any], specs: Mapping[str, Mapping[str, Any]]) -> dict[str, bytes]:
+    source = Path(required_string(object_value(config.get("file"), "file"), "path", "file"))
+    if not source.is_absolute():
+        fail("file.path must be absolute")
+    try:
+        metadata = source.lstat()
+    except OSError as exc:
+        fail(f"cannot inspect fallback secret directory: {exc.__class__.__name__}")
+    if stat.S_ISLNK(metadata.st_mode) or not stat.S_ISDIR(metadata.st_mode):
+        fail("fallback secret directory must be a non-symlink directory")
+    if os.name != "nt" and stat.S_IMODE(metadata.st_mode) & 0o077:
+        fail("fallback secret directory must be mode 0700 or stricter")
+    expected = set(specs)
+    actual = {entry.name for entry in source.iterdir()}
+    if actual != expected:
+        fail("fallback secret directory does not exactly match configured keys")
+    values: dict[str, bytes] = {}
+    for name, spec in specs.items():
+        path = source / name
+        private_file(path, f"fallback secret {name}")
+        value = read_limited(path, int(spec.get("max_bytes", 65_536)), f"fallback secret {name}")
+        if not value or b"\x00" in value:
+            fail(f"fallback secret {name} is empty or unsafe")
+        values[name] = value
+    return values
+
+
+def materialize(runtime: Path, specs: Mapping[str, Mapping[str, Any]], values: Mapping[str, bytes]) -> list[str]:
+    runtime.mkdir(parents=True, exist_ok=True, mode=0o700)
+    if runtime.is_symlink():
+        fail("runtime directory must not be a symlink")
+    os.chmod(runtime, 0o700)
+    paths: dict[str, Path] = {}
+    for name, value in values.items():
+        descriptor, temporary = tempfile.mkstemp(prefix=f".{name}.", dir=runtime)
+        temporary_path = Path(temporary)
+        with os.fdopen(descriptor, "wb") as stream:
+            stream.write(value)
+            stream.flush()
+            os.fsync(stream.fileno())
+        os.chmod(temporary_path, 0o444)
+        destination = runtime / name
+        os.replace(temporary_path, destination)
+        paths[name] = destination
+    consumers = sorted({consumer for spec in specs.values() for consumer in spec["consumers"]})
+    for consumer in consumers:
+        lines = [
+            f'{name}_FILE="{paths[name].resolve()}"\n'
+            for name, spec in sorted(specs.items())
+            if consumer in spec["consumers"]
+        ]
+        destination = runtime / f"{consumer}.env"
+        destination.write_text("".join(lines), encoding="utf-8")
+        os.chmod(destination, 0o600)
+    manifest = runtime / "manifest"
+    manifest.write_text("".join(f"{name}={path.resolve()}\n" for name, path in sorted(paths.items())), encoding="utf-8")
+    os.chmod(manifest, 0o600)
+    return consumers
+
+
+def run(config_path: Path, environ: Mapping[str, str] | None = None) -> list[str]:
+    os.umask(0o077)
+    config = load_json(config_path)
+    if config.get("version") != 1 or config.get("mode") not in {"selectel", "file"}:
+        fail("configuration version/mode is invalid")
+    runtime = Path(required_string(config, "runtime_dir", "configuration"))
+    if not runtime.is_absolute():
+        fail("runtime_dir must be absolute")
+    raw_specs = object_value(config.get("secrets"), "secrets")
+    specs: dict[str, Mapping[str, Any]] = {}
+    for name, spec_value in raw_specs.items():
+        spec = object_value(spec_value, f"secrets.{name}")
+        consumers = spec.get("consumers")
+        if (
+            not NAME_RE.fullmatch(name)
+            or not isinstance(consumers, list)
+            or not consumers
+            or any(not isinstance(item, str) or not CONSUMER_RE.fullmatch(item) for item in consumers)
+        ):
+            fail("secret name or consumer list is invalid")
+        specs[name] = spec
+    environment = os.environ if environ is None else environ
+    values = fetch_values(config, specs, environment) if config["mode"] == "selectel" else file_values(config, specs)
+    return materialize(runtime, specs, values)
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--config", required=True, type=Path)
+    args = parser.parse_args()
+    try:
+        consumers = run(args.config)
+    except (LoaderError, OSError, ValueError) as exc:
+        print(f"secrets-loader: {exc}", file=sys.stderr)
+        return 1
+    print(f"secrets-loader: materialized {len(consumers)} VM2 consumer scopes", file=sys.stderr)
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())