Проект разделен на два репозитория

This commit is contained in:
mi
2026-08-14 15:42:45 +03:00
parent e06a77ee1d
commit bbef7a30c9
521 changed files with 2597 additions and 2302 deletions
@@ -0,0 +1,682 @@
#!/usr/bin/env python3
"""Materialize narrowly scoped service dotenv files from Selectel Secrets Manager."""
from __future__ import annotations
import argparse
import base64
import binascii
import json
import os
import random
import re
import ssl
import stat
import sys
import tempfile
import time
import urllib.error
import urllib.parse
import urllib.request
from dataclasses import dataclass
from pathlib import Path
from typing import Any, Callable, Mapping, NoReturn
DEFAULT_IDENTITY_URL = "https://cloud.api.selcloud.ru/identity/v3/auth/tokens"
MAX_CONFIG_BYTES = 1_048_576
MAX_HTTP_BYTES = 1_048_576
MAX_SECRET_BYTES = 65_536
RETRYABLE_STATUS = frozenset({408, 425, 429, 500, 502, 503, 504})
ENV_NAME_RE = re.compile(r"^[A-Z][A-Z0-9_]*$")
SERVICE_NAME_RE = re.compile(r"^[a-zA-Z0-9][a-zA-Z0-9_.-]*$")
DOTENV_LINE_RE = re.compile(r"^([A-Z][A-Z0-9_]*)=(.*)$")
class LoaderError(Exception):
"""An expected, already-redacted loader failure."""
def fail(message: str) -> NoReturn:
raise LoaderError(message)
def _object(value: Any, label: str) -> dict[str, Any]:
if not isinstance(value, dict):
fail(f"{label} must be an object")
return value
def _only_keys(value: Mapping[str, Any], allowed: set[str], label: str) -> None:
unknown = sorted(set(value) - allowed)
if unknown:
fail(f"{label} contains unsupported fields: {', '.join(unknown)}")
def _required_string(value: Mapping[str, Any], key: str, label: str) -> str:
item = value.get(key)
if not isinstance(item, str) or not item:
fail(f"{label}.{key} must be a non-empty string")
return item
def _bounded_int(value: Any, label: str, minimum: int, maximum: int) -> int:
if isinstance(value, bool) or not isinstance(value, int) or not minimum <= value <= maximum:
fail(f"{label} must be an integer from {minimum} through {maximum}")
return value
def read_limited(path: Path, limit: int, label: str) -> bytes:
try:
with path.open("rb") as stream:
data = stream.read(limit + 1)
except OSError as exc:
fail(f"cannot read {label}: {exc.strerror or exc.__class__.__name__}")
if len(data) > limit:
fail(f"{label} exceeds {limit} bytes")
return data
def require_private_regular_file(path: Path, label: str) -> None:
try:
metadata = path.lstat()
except OSError as exc:
fail(f"cannot inspect {label}: {exc.strerror or exc.__class__.__name__}")
if stat.S_ISLNK(metadata.st_mode) or not stat.S_ISREG(metadata.st_mode):
fail(f"{label} must be a regular file and not a symlink")
if os.name != "nt" and stat.S_IMODE(metadata.st_mode) & 0o077:
fail(f"{label} must not be accessible by group or other users")
def load_json(path: Path) -> dict[str, Any]:
raw = read_limited(path, MAX_CONFIG_BYTES, "configuration")
try:
document = json.loads(raw.decode("utf-8"))
except (UnicodeDecodeError, json.JSONDecodeError):
fail("configuration is not valid UTF-8 JSON")
return _object(document, "configuration")
def credential_value(selectel: Mapping[str, Any], environ: Mapping[str, str]) -> str:
methods = sum(key in selectel for key in ("password_file", "password_env"))
if methods != 1:
fail("selectel must set exactly one of password_file or password_env")
if "password_env" in selectel:
variable = _required_string(selectel, "password_env", "selectel")
if not ENV_NAME_RE.fullmatch(variable):
fail("selectel.password_env is not a valid environment variable name")
value = environ.get(variable)
if value is None or not value:
fail(f"credential environment variable {variable} is not set")
return value
configured = Path(_required_string(selectel, "password_file", "selectel"))
if configured.is_absolute():
path = configured
else:
directory = environ.get("CREDENTIALS_DIRECTORY")
if not directory:
fail("relative password_file requires CREDENTIALS_DIRECTORY")
path = Path(directory) / configured
require_private_regular_file(path, "credential")
raw = read_limited(path, 16_384, "credential")
try:
value = raw.decode("utf-8")
except UnicodeDecodeError:
fail("credential is not valid UTF-8")
value = value.removesuffix("\n").removesuffix("\r")
if not value or "\n" in value or "\r" in value or "\x00" in value:
fail("credential must contain exactly one non-empty text line")
return value
class NoRedirect(urllib.request.HTTPRedirectHandler):
def redirect_request(self, req: Any, fp: Any, code: int, msg: str, headers: Any, newurl: str) -> None:
return None
@dataclass(frozen=True)
class HTTPResult:
status: int
headers: Mapping[str, str]
body: bytes
class HTTPClient:
def __init__(
self,
*,
timeout: float,
retries: int,
max_response_bytes: int,
cafile: str | None = None,
opener: Any | None = None,
sleeper: Callable[[float], None] = time.sleep,
jitter: Callable[[], float] = random.random,
) -> None:
self.timeout = timeout
self.retries = retries
self.max_response_bytes = max_response_bytes
self.sleeper = sleeper
self.jitter = jitter
if opener is None:
try:
context = ssl.create_default_context(cafile=cafile)
except (OSError, ssl.SSLError) as exc:
fail(f"cannot initialize TLS trust store: {exc.__class__.__name__}")
self.opener = urllib.request.build_opener(
urllib.request.HTTPSHandler(context=context), NoRedirect()
)
else:
self.opener = opener
def request(
self,
method: str,
url: str,
*,
headers: Mapping[str, str] | None = None,
body: bytes | None = None,
expected: frozenset[int],
) -> HTTPResult:
parsed = urllib.parse.urlsplit(url)
if parsed.scheme != "https" or not parsed.netloc or parsed.username or parsed.password:
fail("provider endpoint must be an HTTPS URL without embedded credentials")
request = urllib.request.Request(
url, data=body, headers=dict(headers or {}), method=method
)
for attempt in range(self.retries + 1):
try:
with self.opener.open(request, timeout=self.timeout) as response:
status = int(response.status)
content_length = response.headers.get("Content-Length")
if content_length:
try:
if int(content_length) > self.max_response_bytes:
fail("provider response exceeds configured limit")
except ValueError:
fail("provider returned an invalid Content-Length")
response_body = response.read(self.max_response_bytes + 1)
if len(response_body) > self.max_response_bytes:
fail("provider response exceeds configured limit")
if status not in expected:
fail(f"provider request failed with HTTP {status}")
return HTTPResult(status, response.headers, response_body)
except urllib.error.HTTPError as exc:
status = int(exc.code)
if status not in RETRYABLE_STATUS or attempt >= self.retries:
fail(f"provider request failed with HTTP {status}")
except (urllib.error.URLError, TimeoutError, OSError):
if attempt >= self.retries:
fail("provider request failed after retries")
delay = min(8.0, 0.25 * (2**attempt)) * (0.5 + self.jitter())
self.sleeper(delay)
fail("provider request failed")
def parse_json_response(result: HTTPResult, label: str) -> dict[str, Any]:
try:
return _object(json.loads(result.body.decode("utf-8")), label)
except (UnicodeDecodeError, json.JSONDecodeError):
fail(f"{label} is not valid JSON")
def project_token_and_catalog(
client: HTTPClient, selectel: Mapping[str, Any], password: str
) -> tuple[str, list[Any]]:
identity_url = selectel.get("identity_url", DEFAULT_IDENTITY_URL)
if not isinstance(identity_url, str):
fail("selectel.identity_url must be a string")
account_id = _required_string(selectel, "account_id", "selectel")
username = _required_string(selectel, "username", "selectel")
project_name = _required_string(selectel, "project_name", "selectel")
payload = {
"auth": {
"identity": {
"methods": ["password"],
"password": {
"user": {
"name": username,
"domain": {"name": account_id},
"password": password,
}
},
},
"scope": {
"project": {
"name": project_name,
"domain": {"name": account_id},
}
},
}
}
result = client.request(
"POST",
identity_url,
headers={"Content-Type": "application/json", "Accept": "application/json"},
body=json.dumps(payload, separators=(",", ":")).encode("utf-8"),
expected=frozenset({201}),
)
token = result.headers.get("X-Subject-Token")
if not isinstance(token, str) or not token:
fail("identity response omitted X-Subject-Token")
document = parse_json_response(result, "identity response")
token_data = document.get("token")
if not isinstance(token_data, dict):
fail("identity response omitted token metadata")
project = token_data.get("project")
if not isinstance(project, dict) or not project.get("id"):
fail("identity token is not project-scoped")
catalog = token_data.get("catalog")
if not isinstance(catalog, list):
fail("identity response omitted service catalog")
return token, catalog
def secrets_endpoint(catalog: list[Any], region: str, interface: str) -> str:
matches: list[str] = []
for service in catalog:
if not isinstance(service, dict) or service.get("type") != "secrets-manager":
continue
endpoints = service.get("endpoints")
if not isinstance(endpoints, list):
continue
for endpoint in endpoints:
if (
isinstance(endpoint, dict)
and endpoint.get("region") == region
and endpoint.get("interface") == interface
and isinstance(endpoint.get("url"), str)
):
matches.append(endpoint["url"].rstrip("/"))
if len(matches) != 1:
fail("service catalog did not contain exactly one matching Secrets Manager endpoint")
return matches[0]
def decode_secret(document: Mapping[str, Any], name: str, limit: int) -> bytes:
# GET /v1/{name} returns the current value inside ``version`` while
# GET /v1/{name}/versions/{id} returns a version object directly.
payload: Mapping[str, Any] = document
version = document.get("version")
if isinstance(version, dict):
payload = version
encoded = payload.get("value")
if not isinstance(encoded, str):
fail(f"secret {name} response omitted base64 value")
try:
value = base64.b64decode(encoded, validate=True)
except (binascii.Error, ValueError):
fail(f"secret {name} has invalid base64 encoding")
if not value:
fail(f"secret {name} is empty")
if len(value) > limit:
fail(f"secret {name} exceeds its configured limit")
if b"\x00" in value or b"\n" in value or b"\r" in value:
fail(f"secret {name} cannot be represented as a dotenv value")
return value
def fetch_selectel(
config: Mapping[str, Any],
specs: Mapping[str, Mapping[str, Any]],
environ: Mapping[str, str],
client_factory: Callable[..., HTTPClient] = HTTPClient,
) -> dict[str, bytes]:
selectel = _object(config.get("selectel"), "selectel")
_only_keys(
selectel,
{
"account_id",
"username",
"project_name",
"region",
"interface",
"password_file",
"password_env",
"identity_url",
"secrets_url",
"ca_file",
},
"selectel",
)
http = _object(config.get("http", {}), "http")
_only_keys(http, {"timeout_seconds", "retries", "max_response_bytes"}, "http")
timeout = http.get("timeout_seconds", 10)
if isinstance(timeout, bool) or not isinstance(timeout, (int, float)) or not 0.1 <= timeout <= 60:
fail("http.timeout_seconds must be from 0.1 through 60")
retries = _bounded_int(http.get("retries", 3), "http.retries", 0, 8)
response_limit = _bounded_int(
http.get("max_response_bytes", MAX_HTTP_BYTES),
"http.max_response_bytes",
1024,
4 * MAX_HTTP_BYTES,
)
cafile = selectel.get("ca_file")
if cafile is not None and (not isinstance(cafile, str) or not cafile):
fail("selectel.ca_file must be a non-empty string")
client = client_factory(
timeout=float(timeout),
retries=retries,
max_response_bytes=response_limit,
cafile=cafile,
)
password = credential_value(selectel, environ)
token, catalog = project_token_and_catalog(client, selectel, password)
region = _required_string(selectel, "region", "selectel")
interface = selectel.get("interface", "public")
if interface not in {"public", "internal"}:
fail("selectel.interface must be public or internal")
override = selectel.get("secrets_url")
if override is not None and (not isinstance(override, str) or not override):
fail("selectel.secrets_url must be a non-empty string")
base_url = override.rstrip("/") if override else secrets_endpoint(catalog, region, interface)
values: dict[str, bytes] = {}
fetched: dict[tuple[str, int | None], bytes] = {}
for canonical, spec in specs.items():
if "literal" in spec:
values[canonical] = b""
continue
remote = _required_string(spec, "remote", f"secrets.{canonical}")
version = spec.get("version")
version_id: int | None = None
if version is not None:
version_id = _bounded_int(
version, f"secrets.{canonical}.version", 1, 2_147_483_647
)
cache_key = (remote, version_id)
if cache_key not in fetched:
path = f"/v1/{urllib.parse.quote(remote, safe='')}"
if version_id is not None:
path += f"/versions/{version_id}"
try:
result = client.request(
"GET",
base_url + path,
headers={"X-Auth-Token": token, "Accept": "application/json"},
expected=frozenset({200}),
)
document = parse_json_response(result, f"secret {canonical} response")
fetched[cache_key] = decode_secret(
document, canonical, MAX_SECRET_BYTES
)
except LoaderError as exc:
fail(f"cannot load {canonical}: {exc}")
limit = _bounded_int(
spec.get("max_bytes", MAX_SECRET_BYTES),
f"secrets.{canonical}.max_bytes",
1,
MAX_SECRET_BYTES,
)
value = fetched[cache_key]
if len(value) > limit:
fail(f"secret {canonical} exceeds its configured limit")
values[canonical] = value
return values
def parse_dotenv(path: Path, expected: set[str], max_bytes: int) -> dict[str, bytes]:
require_private_regular_file(path, "fallback dotenv")
raw = read_limited(path, max_bytes, "fallback dotenv")
try:
text = raw.decode("utf-8")
except UnicodeDecodeError:
fail("fallback dotenv is not valid UTF-8")
values: dict[str, bytes] = {}
for number, line in enumerate(text.splitlines(), 1):
if not line or line.startswith("#"):
continue
match = DOTENV_LINE_RE.fullmatch(line)
if not match:
fail(f"fallback dotenv has invalid syntax at line {number}")
name, encoded_value = match.groups()
if name not in expected:
fail(f"fallback dotenv contains undeclared key {name}")
if name in values:
fail(f"fallback dotenv contains duplicate key {name}")
if encoded_value.startswith('"'):
try:
decoded = json.loads(encoded_value)
except json.JSONDecodeError:
fail(f"fallback dotenv has invalid quoted value at line {number}")
if not isinstance(decoded, str):
fail(f"fallback dotenv has invalid quoted value at line {number}")
value = decoded.encode("utf-8")
elif encoded_value.startswith("'"):
if len(encoded_value) < 2 or not encoded_value.endswith("'"):
fail(f"fallback dotenv has invalid quoted value at line {number}")
value = encoded_value[1:-1].encode("utf-8")
else:
if any(character.isspace() for character in encoded_value) or any(
character in encoded_value for character in ("'", '"', "`", "$", "\\")
):
fail(f"fallback dotenv requires quoting at line {number}")
value = encoded_value.encode("utf-8")
if not value or b"\x00" in value or b"\n" in value or b"\r" in value:
fail(f"fallback dotenv has an empty or unsafe value for {name}")
values[name] = value
missing = sorted(expected - set(values))
if missing:
fail(f"fallback dotenv is missing declared keys: {', '.join(missing)}")
return values
def validate_specs(config: Mapping[str, Any]) -> dict[str, dict[str, Any]]:
raw_specs = _object(config.get("secrets"), "secrets")
if not raw_specs:
fail("secrets must not be empty")
specs: dict[str, dict[str, Any]] = {}
for canonical, raw_spec in raw_specs.items():
if not isinstance(canonical, str) or not ENV_NAME_RE.fullmatch(canonical):
fail("every canonical secret name must be an uppercase environment name")
spec = _object(raw_spec, f"secrets.{canonical}")
_only_keys(
spec,
{"remote", "consumers", "max_bytes", "version", "literal"},
f"secrets.{canonical}",
)
has_remote = "remote" in spec
has_literal = "literal" in spec
if has_remote == has_literal:
fail(f"secrets.{canonical} must set exactly one of remote or literal")
if has_literal and spec["literal"] != "":
fail(f"secrets.{canonical}.literal may only be an empty string")
consumers = spec.get("consumers")
if not isinstance(consumers, list) or not consumers:
fail(f"secrets.{canonical}.consumers must be a non-empty array")
if len(consumers) != len(set(item for item in consumers if isinstance(item, str))):
fail(f"secrets.{canonical}.consumers contains duplicates or invalid values")
for consumer in consumers:
if not isinstance(consumer, str) or not SERVICE_NAME_RE.fullmatch(consumer):
fail(f"secrets.{canonical}.consumers contains an invalid service name")
specs[canonical] = spec
return specs
def dotenv_quote(value: bytes, name: str) -> str:
try:
text = value.decode("utf-8")
except UnicodeDecodeError:
fail(f"secret {name} is not valid UTF-8")
return json.dumps(text, ensure_ascii=False)
def materialize(runtime_dir: Path, specs: Mapping[str, Mapping[str, Any]], values: Mapping[str, bytes]) -> None:
try:
runtime_dir.mkdir(mode=0o700, parents=True, exist_ok=True)
if runtime_dir.is_symlink():
fail("runtime directory must not be a symlink")
os.chmod(runtime_dir, 0o700)
except OSError as exc:
fail(f"cannot prepare runtime directory: {exc.strerror or exc.__class__.__name__}")
consumers = sorted(
{consumer for spec in specs.values() for consumer in spec["consumers"]}
)
staged: list[tuple[Path, Path]] = []
try:
for consumer in consumers:
lines = [
f"{name}={dotenv_quote(values[name], name)}\n"
for name, spec in sorted(specs.items())
if consumer in spec["consumers"]
]
descriptor, temporary = tempfile.mkstemp(
prefix=f".{consumer}.", suffix=".tmp", dir=runtime_dir
)
temporary_path = Path(temporary)
try:
os.chmod(temporary_path, 0o600)
stream = os.fdopen(descriptor, "w", encoding="utf-8", newline="\n")
descriptor = -1
with stream:
stream.writelines(lines)
stream.flush()
os.fsync(stream.fileno())
except BaseException:
if descriptor >= 0:
os.close(descriptor)
raise
staged.append((temporary_path, runtime_dir / f"{consumer}.env"))
for temporary_path, destination in staged:
os.replace(temporary_path, destination)
value_paths: dict[str, Path] = {}
for name, value in sorted(values.items()):
descriptor, temporary = tempfile.mkstemp(
prefix=f".{name}.", suffix=".tmp", dir=runtime_dir
)
temporary_path = Path(temporary)
try:
# Compose implements local secrets as bind mounts. The protected
# 0700 parent prevents host users from traversing to this 0444
# file while allowing a non-root container UID to read its mount.
os.chmod(temporary_path, 0o444)
with os.fdopen(descriptor, "wb") as stream:
descriptor = -1
stream.write(value)
stream.flush()
os.fsync(stream.fileno())
except BaseException:
if descriptor >= 0:
os.close(descriptor)
temporary_path.unlink(missing_ok=True)
raise
destination = runtime_dir / name
os.replace(temporary_path, destination)
value_paths[name] = destination
manifest_lines = [
f"{name}={path.resolve()}\n" for name, path in sorted(value_paths.items())
]
descriptor, temporary = tempfile.mkstemp(
prefix=".manifest.", suffix=".tmp", dir=runtime_dir
)
manifest_path = Path(temporary)
try:
os.chmod(manifest_path, 0o600)
with os.fdopen(descriptor, "w", encoding="utf-8", newline="\n") as stream:
descriptor = -1
stream.writelines(manifest_lines)
stream.flush()
os.fsync(stream.fileno())
except BaseException:
if descriptor >= 0:
os.close(descriptor)
manifest_path.unlink(missing_ok=True)
raise
os.replace(manifest_path, runtime_dir / "manifest")
if hasattr(os, "O_DIRECTORY"):
directory_fd = os.open(runtime_dir, os.O_RDONLY | os.O_DIRECTORY)
try:
os.fsync(directory_fd)
finally:
os.close(directory_fd)
except OSError as exc:
fail(f"cannot atomically materialize service files: {exc.strerror or exc.__class__.__name__}")
finally:
for temporary_path, _ in staged:
try:
temporary_path.unlink(missing_ok=True)
except OSError:
pass
def run(
config_path: Path,
*,
runtime_override: Path | None = None,
environ: Mapping[str, str] | None = None,
client_factory: Callable[..., HTTPClient] = HTTPClient,
) -> list[str]:
os.umask(0o077)
environment = os.environ if environ is None else environ
config = load_json(config_path)
_only_keys(config, {"version", "mode", "runtime_dir", "http", "selectel", "file", "secrets"}, "configuration")
if config.get("version") != 1:
fail("configuration.version must be 1")
mode = config.get("mode")
if mode not in {"selectel", "file"}:
fail("configuration.mode must explicitly be selectel or file")
specs = validate_specs(config)
if runtime_override is None:
configured_runtime = config.get("runtime_dir")
if not isinstance(configured_runtime, str) or not configured_runtime:
fail("configuration.runtime_dir must be a non-empty string")
runtime_dir = Path(configured_runtime)
else:
runtime_dir = runtime_override
if not runtime_dir.is_absolute():
fail("runtime directory must be an absolute path")
if mode == "selectel":
if "file" in config:
fail("file settings are forbidden in selectel mode")
values = fetch_selectel(config, specs, environment, client_factory)
else:
if "selectel" in config or "http" in config:
fail("selectel and http settings are forbidden in file mode")
file_config = _object(config.get("file"), "file")
_only_keys(file_config, {"path", "max_bytes"}, "file")
source = Path(_required_string(file_config, "path", "file"))
if not source.is_absolute():
fail("file.path must be absolute")
max_bytes = _bounded_int(
file_config.get("max_bytes", MAX_CONFIG_BYTES),
"file.max_bytes",
1,
4 * MAX_CONFIG_BYTES,
)
expected = {name for name, spec in specs.items() if "literal" not in spec}
values = parse_dotenv(source, expected, max_bytes)
values.update(
{name: b"" for name, spec in specs.items() if "literal" in spec}
)
for canonical, value in values.items():
limit = _bounded_int(
specs[canonical].get("max_bytes", MAX_SECRET_BYTES),
f"secrets.{canonical}.max_bytes",
1,
MAX_SECRET_BYTES,
)
if len(value) > limit:
fail(f"secret {canonical} exceeds its configured limit")
materialize(runtime_dir, specs, values)
return sorted({consumer for spec in specs.values() for consumer in spec["consumers"]})
def main(argv: list[str] | None = None) -> int:
parser = argparse.ArgumentParser(description="Materialize per-service secret dotenv files")
parser.add_argument("--config", required=True, type=Path)
parser.add_argument("--runtime-dir", type=Path)
arguments = parser.parse_args(argv)
try:
consumers = run(arguments.config, runtime_override=arguments.runtime_dir)
except LoaderError as exc:
print(f"secrets-loader: {exc}", file=sys.stderr)
return 1
print(f"secrets-loader: materialized {len(consumers)} service file(s)", file=sys.stderr)
return 0
if __name__ == "__main__":
raise SystemExit(main())