#!/usr/bin/env python3 """Materialize narrowly scoped service dotenv files from Selectel Secrets Manager.""" from __future__ import annotations import argparse import base64 import binascii import json import os import random import re import ssl import stat import sys import tempfile import time import urllib.error import urllib.parse import urllib.request from dataclasses import dataclass from pathlib import Path from typing import Any, Callable, Mapping, NoReturn DEFAULT_IDENTITY_URL = "https://cloud.api.selcloud.ru/identity/v3/auth/tokens" MAX_CONFIG_BYTES = 1_048_576 MAX_HTTP_BYTES = 1_048_576 MAX_SECRET_BYTES = 65_536 RETRYABLE_STATUS = frozenset({408, 425, 429, 500, 502, 503, 504}) ENV_NAME_RE = re.compile(r"^[A-Z][A-Z0-9_]*$") SERVICE_NAME_RE = re.compile(r"^[a-zA-Z0-9][a-zA-Z0-9_.-]*$") DOTENV_LINE_RE = re.compile(r"^([A-Z][A-Z0-9_]*)=(.*)$") class LoaderError(Exception): """An expected, already-redacted loader failure.""" def fail(message: str) -> NoReturn: raise LoaderError(message) def _object(value: Any, label: str) -> dict[str, Any]: if not isinstance(value, dict): fail(f"{label} must be an object") return value def _only_keys(value: Mapping[str, Any], allowed: set[str], label: str) -> None: unknown = sorted(set(value) - allowed) if unknown: fail(f"{label} contains unsupported fields: {', '.join(unknown)}") def _required_string(value: Mapping[str, Any], key: str, label: str) -> str: item = value.get(key) if not isinstance(item, str) or not item: fail(f"{label}.{key} must be a non-empty string") return item def _bounded_int(value: Any, label: str, minimum: int, maximum: int) -> int: if isinstance(value, bool) or not isinstance(value, int) or not minimum <= value <= maximum: fail(f"{label} must be an integer from {minimum} through {maximum}") return value def read_limited(path: Path, limit: int, label: str) -> bytes: try: with path.open("rb") as stream: data = stream.read(limit + 1) except OSError as exc: fail(f"cannot read {label}: {exc.strerror or exc.__class__.__name__}") if len(data) > limit: fail(f"{label} exceeds {limit} bytes") return data def require_private_regular_file(path: Path, label: str) -> None: try: metadata = path.lstat() except OSError as exc: fail(f"cannot inspect {label}: {exc.strerror or exc.__class__.__name__}") if stat.S_ISLNK(metadata.st_mode) or not stat.S_ISREG(metadata.st_mode): fail(f"{label} must be a regular file and not a symlink") if os.name != "nt" and stat.S_IMODE(metadata.st_mode) & 0o077: fail(f"{label} must not be accessible by group or other users") def load_json(path: Path) -> dict[str, Any]: raw = read_limited(path, MAX_CONFIG_BYTES, "configuration") try: document = json.loads(raw.decode("utf-8")) except (UnicodeDecodeError, json.JSONDecodeError): fail("configuration is not valid UTF-8 JSON") return _object(document, "configuration") def credential_value(selectel: Mapping[str, Any], environ: Mapping[str, str]) -> str: methods = sum(key in selectel for key in ("password_file", "password_env")) if methods != 1: fail("selectel must set exactly one of password_file or password_env") if "password_env" in selectel: variable = _required_string(selectel, "password_env", "selectel") if not ENV_NAME_RE.fullmatch(variable): fail("selectel.password_env is not a valid environment variable name") value = environ.get(variable) if value is None or not value: fail(f"credential environment variable {variable} is not set") return value configured = Path(_required_string(selectel, "password_file", "selectel")) if configured.is_absolute(): path = configured else: directory = environ.get("CREDENTIALS_DIRECTORY") if not directory: fail("relative password_file requires CREDENTIALS_DIRECTORY") path = Path(directory) / configured require_private_regular_file(path, "credential") raw = read_limited(path, 16_384, "credential") try: value = raw.decode("utf-8") except UnicodeDecodeError: fail("credential is not valid UTF-8") value = value.removesuffix("\n").removesuffix("\r") if not value or "\n" in value or "\r" in value or "\x00" in value: fail("credential must contain exactly one non-empty text line") return value class NoRedirect(urllib.request.HTTPRedirectHandler): def redirect_request(self, req: Any, fp: Any, code: int, msg: str, headers: Any, newurl: str) -> None: return None @dataclass(frozen=True) class HTTPResult: status: int headers: Mapping[str, str] body: bytes class HTTPClient: def __init__( self, *, timeout: float, retries: int, max_response_bytes: int, cafile: str | None = None, opener: Any | None = None, sleeper: Callable[[float], None] = time.sleep, jitter: Callable[[], float] = random.random, ) -> None: self.timeout = timeout self.retries = retries self.max_response_bytes = max_response_bytes self.sleeper = sleeper self.jitter = jitter if opener is None: try: context = ssl.create_default_context(cafile=cafile) except (OSError, ssl.SSLError) as exc: fail(f"cannot initialize TLS trust store: {exc.__class__.__name__}") self.opener = urllib.request.build_opener( urllib.request.HTTPSHandler(context=context), NoRedirect() ) else: self.opener = opener def request( self, method: str, url: str, *, headers: Mapping[str, str] | None = None, body: bytes | None = None, expected: frozenset[int], ) -> HTTPResult: parsed = urllib.parse.urlsplit(url) if parsed.scheme != "https" or not parsed.netloc or parsed.username or parsed.password: fail("provider endpoint must be an HTTPS URL without embedded credentials") request = urllib.request.Request( url, data=body, headers=dict(headers or {}), method=method ) for attempt in range(self.retries + 1): try: with self.opener.open(request, timeout=self.timeout) as response: status = int(response.status) content_length = response.headers.get("Content-Length") if content_length: try: if int(content_length) > self.max_response_bytes: fail("provider response exceeds configured limit") except ValueError: fail("provider returned an invalid Content-Length") response_body = response.read(self.max_response_bytes + 1) if len(response_body) > self.max_response_bytes: fail("provider response exceeds configured limit") if status not in expected: fail(f"provider request failed with HTTP {status}") return HTTPResult(status, response.headers, response_body) except urllib.error.HTTPError as exc: status = int(exc.code) if status not in RETRYABLE_STATUS or attempt >= self.retries: fail(f"provider request failed with HTTP {status}") except (urllib.error.URLError, TimeoutError, OSError): if attempt >= self.retries: fail("provider request failed after retries") delay = min(8.0, 0.25 * (2**attempt)) * (0.5 + self.jitter()) self.sleeper(delay) fail("provider request failed") def parse_json_response(result: HTTPResult, label: str) -> dict[str, Any]: try: return _object(json.loads(result.body.decode("utf-8")), label) except (UnicodeDecodeError, json.JSONDecodeError): fail(f"{label} is not valid JSON") def project_token_and_catalog( client: HTTPClient, selectel: Mapping[str, Any], password: str ) -> tuple[str, list[Any]]: identity_url = selectel.get("identity_url", DEFAULT_IDENTITY_URL) if not isinstance(identity_url, str): fail("selectel.identity_url must be a string") account_id = _required_string(selectel, "account_id", "selectel") username = _required_string(selectel, "username", "selectel") project_name = _required_string(selectel, "project_name", "selectel") payload = { "auth": { "identity": { "methods": ["password"], "password": { "user": { "name": username, "domain": {"name": account_id}, "password": password, } }, }, "scope": { "project": { "name": project_name, "domain": {"name": account_id}, } }, } } result = client.request( "POST", identity_url, headers={"Content-Type": "application/json", "Accept": "application/json"}, body=json.dumps(payload, separators=(",", ":")).encode("utf-8"), expected=frozenset({201}), ) token = result.headers.get("X-Subject-Token") if not isinstance(token, str) or not token: fail("identity response omitted X-Subject-Token") document = parse_json_response(result, "identity response") token_data = document.get("token") if not isinstance(token_data, dict): fail("identity response omitted token metadata") project = token_data.get("project") if not isinstance(project, dict) or not project.get("id"): fail("identity token is not project-scoped") catalog = token_data.get("catalog") if not isinstance(catalog, list): fail("identity response omitted service catalog") return token, catalog def secrets_endpoint(catalog: list[Any], region: str, interface: str) -> str: matches: list[str] = [] for service in catalog: if not isinstance(service, dict) or service.get("type") != "secrets-manager": continue endpoints = service.get("endpoints") if not isinstance(endpoints, list): continue for endpoint in endpoints: if ( isinstance(endpoint, dict) and endpoint.get("region") == region and endpoint.get("interface") == interface and isinstance(endpoint.get("url"), str) ): matches.append(endpoint["url"].rstrip("/")) if len(matches) != 1: fail("service catalog did not contain exactly one matching Secrets Manager endpoint") return matches[0] def decode_secret(document: Mapping[str, Any], name: str, limit: int) -> bytes: # GET /v1/{name} returns the current value inside ``version`` while # GET /v1/{name}/versions/{id} returns a version object directly. payload: Mapping[str, Any] = document version = document.get("version") if isinstance(version, dict): payload = version encoded = payload.get("value") if not isinstance(encoded, str): fail(f"secret {name} response omitted base64 value") try: value = base64.b64decode(encoded, validate=True) except (binascii.Error, ValueError): fail(f"secret {name} has invalid base64 encoding") if not value: fail(f"secret {name} is empty") if len(value) > limit: fail(f"secret {name} exceeds its configured limit") if b"\x00" in value or b"\n" in value or b"\r" in value: fail(f"secret {name} cannot be represented as a dotenv value") return value def fetch_selectel( config: Mapping[str, Any], specs: Mapping[str, Mapping[str, Any]], environ: Mapping[str, str], client_factory: Callable[..., HTTPClient] = HTTPClient, ) -> dict[str, bytes]: selectel = _object(config.get("selectel"), "selectel") _only_keys( selectel, { "account_id", "username", "project_name", "region", "interface", "password_file", "password_env", "identity_url", "secrets_url", "ca_file", }, "selectel", ) http = _object(config.get("http", {}), "http") _only_keys(http, {"timeout_seconds", "retries", "max_response_bytes"}, "http") timeout = http.get("timeout_seconds", 10) if isinstance(timeout, bool) or not isinstance(timeout, (int, float)) or not 0.1 <= timeout <= 60: fail("http.timeout_seconds must be from 0.1 through 60") retries = _bounded_int(http.get("retries", 3), "http.retries", 0, 8) response_limit = _bounded_int( http.get("max_response_bytes", MAX_HTTP_BYTES), "http.max_response_bytes", 1024, 4 * MAX_HTTP_BYTES, ) cafile = selectel.get("ca_file") if cafile is not None and (not isinstance(cafile, str) or not cafile): fail("selectel.ca_file must be a non-empty string") client = client_factory( timeout=float(timeout), retries=retries, max_response_bytes=response_limit, cafile=cafile, ) password = credential_value(selectel, environ) token, catalog = project_token_and_catalog(client, selectel, password) region = _required_string(selectel, "region", "selectel") interface = selectel.get("interface", "public") if interface not in {"public", "internal"}: fail("selectel.interface must be public or internal") override = selectel.get("secrets_url") if override is not None and (not isinstance(override, str) or not override): fail("selectel.secrets_url must be a non-empty string") base_url = override.rstrip("/") if override else secrets_endpoint(catalog, region, interface) values: dict[str, bytes] = {} fetched: dict[tuple[str, int | None], bytes] = {} for canonical, spec in specs.items(): if "literal" in spec: values[canonical] = b"" continue remote = _required_string(spec, "remote", f"secrets.{canonical}") version = spec.get("version") version_id: int | None = None if version is not None: version_id = _bounded_int( version, f"secrets.{canonical}.version", 1, 2_147_483_647 ) cache_key = (remote, version_id) if cache_key not in fetched: path = f"/v1/{urllib.parse.quote(remote, safe='')}" if version_id is not None: path += f"/versions/{version_id}" try: result = client.request( "GET", base_url + path, headers={"X-Auth-Token": token, "Accept": "application/json"}, expected=frozenset({200}), ) document = parse_json_response(result, f"secret {canonical} response") fetched[cache_key] = decode_secret( document, canonical, MAX_SECRET_BYTES ) except LoaderError as exc: fail(f"cannot load {canonical}: {exc}") limit = _bounded_int( spec.get("max_bytes", MAX_SECRET_BYTES), f"secrets.{canonical}.max_bytes", 1, MAX_SECRET_BYTES, ) value = fetched[cache_key] if len(value) > limit: fail(f"secret {canonical} exceeds its configured limit") values[canonical] = value return values def parse_dotenv(path: Path, expected: set[str], max_bytes: int) -> dict[str, bytes]: require_private_regular_file(path, "fallback dotenv") raw = read_limited(path, max_bytes, "fallback dotenv") try: text = raw.decode("utf-8") except UnicodeDecodeError: fail("fallback dotenv is not valid UTF-8") values: dict[str, bytes] = {} for number, line in enumerate(text.splitlines(), 1): if not line or line.startswith("#"): continue match = DOTENV_LINE_RE.fullmatch(line) if not match: fail(f"fallback dotenv has invalid syntax at line {number}") name, encoded_value = match.groups() if name not in expected: fail(f"fallback dotenv contains undeclared key {name}") if name in values: fail(f"fallback dotenv contains duplicate key {name}") if encoded_value.startswith('"'): try: decoded = json.loads(encoded_value) except json.JSONDecodeError: fail(f"fallback dotenv has invalid quoted value at line {number}") if not isinstance(decoded, str): fail(f"fallback dotenv has invalid quoted value at line {number}") value = decoded.encode("utf-8") elif encoded_value.startswith("'"): if len(encoded_value) < 2 or not encoded_value.endswith("'"): fail(f"fallback dotenv has invalid quoted value at line {number}") value = encoded_value[1:-1].encode("utf-8") else: if any(character.isspace() for character in encoded_value) or any( character in encoded_value for character in ("'", '"', "`", "$", "\\") ): fail(f"fallback dotenv requires quoting at line {number}") value = encoded_value.encode("utf-8") if not value or b"\x00" in value or b"\n" in value or b"\r" in value: fail(f"fallback dotenv has an empty or unsafe value for {name}") values[name] = value missing = sorted(expected - set(values)) if missing: fail(f"fallback dotenv is missing declared keys: {', '.join(missing)}") return values def validate_specs(config: Mapping[str, Any]) -> dict[str, dict[str, Any]]: raw_specs = _object(config.get("secrets"), "secrets") if not raw_specs: fail("secrets must not be empty") specs: dict[str, dict[str, Any]] = {} for canonical, raw_spec in raw_specs.items(): if not isinstance(canonical, str) or not ENV_NAME_RE.fullmatch(canonical): fail("every canonical secret name must be an uppercase environment name") spec = _object(raw_spec, f"secrets.{canonical}") _only_keys( spec, {"remote", "consumers", "max_bytes", "version", "literal"}, f"secrets.{canonical}", ) has_remote = "remote" in spec has_literal = "literal" in spec if has_remote == has_literal: fail(f"secrets.{canonical} must set exactly one of remote or literal") if has_literal and spec["literal"] != "": fail(f"secrets.{canonical}.literal may only be an empty string") consumers = spec.get("consumers") if not isinstance(consumers, list) or not consumers: fail(f"secrets.{canonical}.consumers must be a non-empty array") if len(consumers) != len(set(item for item in consumers if isinstance(item, str))): fail(f"secrets.{canonical}.consumers contains duplicates or invalid values") for consumer in consumers: if not isinstance(consumer, str) or not SERVICE_NAME_RE.fullmatch(consumer): fail(f"secrets.{canonical}.consumers contains an invalid service name") specs[canonical] = spec return specs def dotenv_quote(value: bytes, name: str) -> str: try: text = value.decode("utf-8") except UnicodeDecodeError: fail(f"secret {name} is not valid UTF-8") return json.dumps(text, ensure_ascii=False) def materialize(runtime_dir: Path, specs: Mapping[str, Mapping[str, Any]], values: Mapping[str, bytes]) -> None: try: runtime_dir.mkdir(mode=0o700, parents=True, exist_ok=True) if runtime_dir.is_symlink(): fail("runtime directory must not be a symlink") os.chmod(runtime_dir, 0o700) except OSError as exc: fail(f"cannot prepare runtime directory: {exc.strerror or exc.__class__.__name__}") consumers = sorted( {consumer for spec in specs.values() for consumer in spec["consumers"]} ) staged: list[tuple[Path, Path]] = [] try: for consumer in consumers: lines = [ f"{name}={dotenv_quote(values[name], name)}\n" for name, spec in sorted(specs.items()) if consumer in spec["consumers"] ] descriptor, temporary = tempfile.mkstemp( prefix=f".{consumer}.", suffix=".tmp", dir=runtime_dir ) temporary_path = Path(temporary) try: os.chmod(temporary_path, 0o600) stream = os.fdopen(descriptor, "w", encoding="utf-8", newline="\n") descriptor = -1 with stream: stream.writelines(lines) stream.flush() os.fsync(stream.fileno()) except BaseException: if descriptor >= 0: os.close(descriptor) raise staged.append((temporary_path, runtime_dir / f"{consumer}.env")) for temporary_path, destination in staged: os.replace(temporary_path, destination) value_paths: dict[str, Path] = {} for name, value in sorted(values.items()): descriptor, temporary = tempfile.mkstemp( prefix=f".{name}.", suffix=".tmp", dir=runtime_dir ) temporary_path = Path(temporary) try: # Compose implements local secrets as bind mounts. The protected # 0700 parent prevents host users from traversing to this 0444 # file while allowing a non-root container UID to read its mount. os.chmod(temporary_path, 0o444) with os.fdopen(descriptor, "wb") as stream: descriptor = -1 stream.write(value) stream.flush() os.fsync(stream.fileno()) except BaseException: if descriptor >= 0: os.close(descriptor) temporary_path.unlink(missing_ok=True) raise destination = runtime_dir / name os.replace(temporary_path, destination) value_paths[name] = destination manifest_lines = [ f"{name}={path.resolve()}\n" for name, path in sorted(value_paths.items()) ] descriptor, temporary = tempfile.mkstemp( prefix=".manifest.", suffix=".tmp", dir=runtime_dir ) manifest_path = Path(temporary) try: os.chmod(manifest_path, 0o600) with os.fdopen(descriptor, "w", encoding="utf-8", newline="\n") as stream: descriptor = -1 stream.writelines(manifest_lines) stream.flush() os.fsync(stream.fileno()) except BaseException: if descriptor >= 0: os.close(descriptor) manifest_path.unlink(missing_ok=True) raise os.replace(manifest_path, runtime_dir / "manifest") if hasattr(os, "O_DIRECTORY"): directory_fd = os.open(runtime_dir, os.O_RDONLY | os.O_DIRECTORY) try: os.fsync(directory_fd) finally: os.close(directory_fd) except OSError as exc: fail(f"cannot atomically materialize service files: {exc.strerror or exc.__class__.__name__}") finally: for temporary_path, _ in staged: try: temporary_path.unlink(missing_ok=True) except OSError: pass def run( config_path: Path, *, runtime_override: Path | None = None, environ: Mapping[str, str] | None = None, client_factory: Callable[..., HTTPClient] = HTTPClient, ) -> list[str]: os.umask(0o077) environment = os.environ if environ is None else environ config = load_json(config_path) _only_keys(config, {"version", "mode", "runtime_dir", "http", "selectel", "file", "secrets"}, "configuration") if config.get("version") != 1: fail("configuration.version must be 1") mode = config.get("mode") if mode not in {"selectel", "file"}: fail("configuration.mode must explicitly be selectel or file") specs = validate_specs(config) if runtime_override is None: configured_runtime = config.get("runtime_dir") if not isinstance(configured_runtime, str) or not configured_runtime: fail("configuration.runtime_dir must be a non-empty string") runtime_dir = Path(configured_runtime) else: runtime_dir = runtime_override if not runtime_dir.is_absolute(): fail("runtime directory must be an absolute path") if mode == "selectel": if "file" in config: fail("file settings are forbidden in selectel mode") values = fetch_selectel(config, specs, environment, client_factory) else: if "selectel" in config or "http" in config: fail("selectel and http settings are forbidden in file mode") file_config = _object(config.get("file"), "file") _only_keys(file_config, {"path", "max_bytes"}, "file") source = Path(_required_string(file_config, "path", "file")) if not source.is_absolute(): fail("file.path must be absolute") max_bytes = _bounded_int( file_config.get("max_bytes", MAX_CONFIG_BYTES), "file.max_bytes", 1, 4 * MAX_CONFIG_BYTES, ) expected = {name for name, spec in specs.items() if "literal" not in spec} values = parse_dotenv(source, expected, max_bytes) values.update( {name: b"" for name, spec in specs.items() if "literal" in spec} ) for canonical, value in values.items(): limit = _bounded_int( specs[canonical].get("max_bytes", MAX_SECRET_BYTES), f"secrets.{canonical}.max_bytes", 1, MAX_SECRET_BYTES, ) if len(value) > limit: fail(f"secret {canonical} exceeds its configured limit") materialize(runtime_dir, specs, values) return sorted({consumer for spec in specs.values() for consumer in spec["consumers"]}) def main(argv: list[str] | None = None) -> int: parser = argparse.ArgumentParser(description="Materialize per-service secret dotenv files") parser.add_argument("--config", required=True, type=Path) parser.add_argument("--runtime-dir", type=Path) arguments = parser.parse_args(argv) try: consumers = run(arguments.config, runtime_override=arguments.runtime_dir) except LoaderError as exc: print(f"secrets-loader: {exc}", file=sys.stderr) return 1 print(f"secrets-loader: materialized {len(consumers)} service file(s)", file=sys.stderr) return 0 if __name__ == "__main__": raise SystemExit(main())