From c149a1f89184aa56c998ce3cf6ff8c0a908a9083 Mon Sep 17 00:00:00 2001 From: Paulo Date: Sun, 4 Oct 2026 14:45:30 +0200 Subject: [PATCH] DRU-735 -- Keep secrets out of druks.toml --- .env.example | 8 +- README.md | 5 +- backend/druks/alembic_support.py | 2 +- backend/druks/api/server.py | 2 +- backend/druks/browser/login.py | 2 +- backend/druks/cli.py | 4 +- backend/druks/database.py | 2 +- backend/druks/doctor.py | 10 +- backend/druks/durable/engine.py | 4 +- backend/druks/harnesses/datastructures.py | 4 +- backend/druks/redis.py | 2 +- backend/druks/sandbox/client.py | 2 +- backend/druks/settings.py | 148 +++++++++++++--- backend/druks/setup_env.py | 203 +++++++++++++--------- backend/druks/testing.py | 14 +- backend/tests/test_browser_sessions.py | 2 +- backend/tests/test_secrets.py | 36 ++-- backend/tests/test_settings.py | 100 ++++++++--- backend/tests/test_setup_env.py | 104 ++++++++--- docs/configuration.md | 107 +++++++++--- docs/deployment.md | 19 +- docs/development.md | 5 +- docs/full-local.md | 2 +- docs/troubleshooting.md | 3 +- druks.toml.example | 8 +- scripts/install.sh | 2 +- 26 files changed, 552 insertions(+), 248 deletions(-) diff --git a/.env.example b/.env.example index 7dd3868ce..073f90f1f 100644 --- a/.env.example +++ b/.env.example @@ -5,6 +5,10 @@ # for the app and a separate druks database rebuilt by pytest. DRUKS_DATABASE_URL=postgresql+psycopg://druks:druks@localhost:5432/druks_dev +# Encrypts stored secrets at rest. Generate one: +# python3 -c 'import base64, os; print(base64.b64encode(os.urandom(32)).decode())' +DRUKS_SECRETS_KEY= + # Short-lived coordination: webhook claims, OAuth state/token caches, and the # sandbox provisioning gate. Workflow state lives in Postgres, not Redis. DRUKS_REDIS_URL=redis://127.0.0.1:6379/0 @@ -15,10 +19,6 @@ DRUKS_TEST_REDIS_URL=redis://127.0.0.1:6379/15 DRUKS_DATA_DIR=~/.druks DRUKS_LOG_LEVEL=INFO -# Optional Slack interactivity authentication. Destination webhook URLs are -# configured in the dashboard. -SLACK_SIGNING_SECRET= - DRUKS_SANDBOX_KEYS_DIR=~/.druks/sandbox-keys # Optional harness configuration copied into sandboxes. Provider credentials diff --git a/README.md b/README.md index 8e49a2f6c..64e376415 100644 --- a/README.md +++ b/README.md @@ -65,12 +65,13 @@ curl -fsSL https://druks.ai/install.sh | DRUKS_PROVIDER=exe bash ``` The installer does not ask questions. The first run writes -`~/druks/druks.toml` with generated secrets. A remote shape can require values +`~/druks/druks.toml`, and `~/druks/.env` with generated secrets. A remote shape +can require values that only you know. These values include provider credentials and identity-edge details. The installer prints this list and exits. Set the values in -`druks.toml`. Then run the same command again. +`druks.toml` and `.env`. Then run the same command again. See the [deployment runbook](https://docs.druks.ai/deployment) for prerequisites, access control, verification, and rollback. diff --git a/backend/druks/alembic_support.py b/backend/druks/alembic_support.py index 3e68cb59d..15eb6879d 100644 --- a/backend/druks/alembic_support.py +++ b/backend/druks/alembic_support.py @@ -36,7 +36,7 @@ def run_alembic_env(target_metadata=None) -> None: if target_metadata is None: target_metadata = config.attributes.get("target_metadata", Base.metadata) if not config.get_main_option("sqlalchemy.url"): - config.set_main_option("sqlalchemy.url", load_settings().database_url) + config.set_main_option("sqlalchemy.url", load_settings().database_url.get_secret_value()) def include_object(obj, name, type_, reflected, compare_to): target = compare_to if compare_to is not None else obj diff --git a/backend/druks/api/server.py b/backend/druks/api/server.py index c65dec673..f332bad7d 100644 --- a/backend/druks/api/server.py +++ b/backend/druks/api/server.py @@ -65,7 +65,7 @@ def configure_state(app: FastAPI, settings: Settings) -> None: ensure_data_dirs(settings) app.state.settings = settings - app.state.engine = create_async_engine_from_url(settings.database_url) + app.state.engine = create_async_engine_from_url(settings.database_url.get_secret_value()) # Bind the ambient (``scoped_session``) factory to this engine so # request handlers can use ``db_session()`` without per-call setup. configure_session(app.state.engine) diff --git a/backend/druks/browser/login.py b/backend/druks/browser/login.py index ea9ed01b3..ccd2e2cb8 100644 --- a/backend/druks/browser/login.py +++ b/backend/druks/browser/login.py @@ -57,7 +57,7 @@ async def open(cls, session: StoredBrowserSession) -> "LoginWindow": browser, session.name, start_url=f"https://{session.site}", - login_proxy=settings.sandbox.browser_login_proxy, + login_proxy=settings.sandbox.browser_login_proxy.get_secret_value(), login_tz=settings.sandbox.browser_login_tz, ) except BaseException: diff --git a/backend/druks/cli.py b/backend/druks/cli.py index d87d44da0..e303f7431 100644 --- a/backend/druks/cli.py +++ b/backend/druks/cli.py @@ -140,11 +140,11 @@ def main() -> None: ensure_data_dirs(settings) if args.command == "init-db": - run_migrations(settings.database_url) + run_migrations(settings.database_url.get_secret_value()) return if args.command == "makemigrations": - make_app_migration(args.app, args.message, settings.database_url) + make_app_migration(args.app, args.message, settings.database_url.get_secret_value()) return raise AssertionError(f"Unhandled command: {args.command}") diff --git a/backend/druks/database.py b/backend/druks/database.py index 4fd793d0f..5aa6c4e31 100644 --- a/backend/druks/database.py +++ b/backend/druks/database.py @@ -95,7 +95,7 @@ def _app_migration_dirs() -> list[tuple[str, Path]]: # Every encrypted column reads its keys from the settings at each use. The # HKDF info predates the library and must never change: stored rows carry it. -configure(lambda: load_settings().secrets.secrets_key, info=b"druks-secrets-v1") +configure(lambda: load_settings().secrets_key.get_secret_value(), info=b"druks-secrets-v1") def create_engine_from_url(database_url: str): diff --git a/backend/druks/doctor.py b/backend/druks/doctor.py index d6dc9f7e7..b97925a3e 100644 --- a/backend/druks/doctor.py +++ b/backend/druks/doctor.py @@ -54,7 +54,7 @@ class CheckResult: @asynccontextmanager async def _check_engine(settings: Settings): # The suite patches this to hand in the fixture's connection. - engine = create_async_engine_from_url(settings.database_url) + engine = create_async_engine_from_url(settings.database_url.get_secret_value()) try: yield engine finally: @@ -166,7 +166,7 @@ def _credentials_check( def check_provider_credentials(settings: Settings) -> list[CheckResult]: # One result per registered provider. Doctor is a one-off, so a plain # session reads the rows and never binds the ambient registry. - engine = create_engine_from_url(settings.database_url) + engine = create_engine_from_url(settings.database_url.get_secret_value()) try: with Session(engine) as session: default_account_id = session.scalar(select(Account.id).where(Account.is_default)) @@ -251,7 +251,7 @@ def check_data_dir(settings: Settings) -> CheckResult: def check_database(settings: Settings) -> CheckResult: try: - engine = create_engine_from_url(settings.database_url) + engine = create_engine_from_url(settings.database_url.get_secret_value()) with engine.connect() as conn: conn.execute(text("SELECT 1")) engine.dispose() @@ -273,7 +273,7 @@ async def check_drukbox(settings: Settings) -> CheckResult: ) api = SandboxAPI( base_url=settings.sandbox.service_url, - token=settings.sandbox.service_token, + token=settings.sandbox.service_token.get_secret_value(), timeout=settings.sandbox.timeout, ) try: @@ -400,7 +400,7 @@ async def _doctor_exec(sandbox) -> None: def check_redis(settings: Settings) -> CheckResult: - parsed = urlparse(settings.redis_url) + parsed = urlparse(settings.redis_url.get_secret_value()) host = parsed.hostname or "127.0.0.1" port = parsed.port or 6379 try: diff --git a/backend/druks/durable/engine.py b/backend/druks/durable/engine.py index 3a79d9976..8f8a16fb3 100644 --- a/backend/druks/durable/engine.py +++ b/backend/druks/durable/engine.py @@ -40,7 +40,7 @@ def init_dbos() -> None: settings = load_settings() # System URL is the app database: DBOS self-migrates its bookkeeping into # the dbos schema there, so derived Run.state is a same-DB read. - url = _dbos_database_url(settings.database_url) + url = _dbos_database_url(settings.database_url.get_secret_value()) config: DBOSConfig = { "name": "druks", "system_database_url": url, @@ -164,7 +164,7 @@ def configure_engine(engine) -> None: def _step_engine(): global _engine if not _engine: - _engine = create_async_engine_from_url(load_settings().database_url) + _engine = create_async_engine_from_url(load_settings().database_url.get_secret_value()) return _engine diff --git a/backend/druks/harnesses/datastructures.py b/backend/druks/harnesses/datastructures.py index 932ac5105..53e047e47 100644 --- a/backend/druks/harnesses/datastructures.py +++ b/backend/druks/harnesses/datastructures.py @@ -4,6 +4,8 @@ from pathlib import Path from typing import Literal, Self +from pydantic import SecretStr + # Execution-side types live with the executor; re-exported here # because the harness API speaks them. from druks.sandbox.datastructures import ( # noqa: F401 @@ -101,7 +103,7 @@ class ParsedUsage: @dataclass(frozen=True) class SandboxSettings: service_url: str - service_token: str + service_token: SecretStr service_timeout: float image: str # Each harness owns one directory under this root. Missing files are not diff --git a/backend/druks/redis.py b/backend/druks/redis.py index 939311d36..54cde3338 100644 --- a/backend/druks/redis.py +++ b/backend/druks/redis.py @@ -8,7 +8,7 @@ def get_client() -> aioredis.Redis: global _client if not _client: - _client = aioredis.from_url(load_settings().redis_url) + _client = aioredis.from_url(load_settings().redis_url.get_secret_value()) return _client diff --git a/backend/druks/sandbox/client.py b/backend/druks/sandbox/client.py index f54ed9014..a347ad679 100644 --- a/backend/druks/sandbox/client.py +++ b/backend/druks/sandbox/client.py @@ -331,7 +331,7 @@ def _api(self) -> SandboxAPI: settings = load_settings() return SandboxAPI( base_url=settings.sandbox.service_url, - token=settings.sandbox.service_token, + token=settings.sandbox.service_token.get_secret_value(), timeout=settings.sandbox.timeout, ) diff --git a/backend/druks/settings.py b/backend/druks/settings.py index c90cb2117..bfef615b8 100644 --- a/backend/druks/settings.py +++ b/backend/druks/settings.py @@ -1,13 +1,24 @@ import logging import os +from collections.abc import Mapping from pathlib import Path from typing import Annotated, Any, Literal import asyncssh from jsonpointer import JsonPointer, JsonPointerException -from pydantic import AfterValidator, BaseModel, BeforeValidator, Field, model_validator +from pydantic import ( + AfterValidator, + BaseModel, + BeforeValidator, + Field, + SecretStr, + model_validator, +) from pydantic_settings import ( BaseSettings, + DotEnvSettingsSource, + EnvSettingsSource, + NestedSecretsSettingsSource, PydanticBaseSettingsSource, SettingsConfigDict, TomlConfigSettingsSource, @@ -39,7 +50,7 @@ def _expand_optional_path(value: Any) -> Any: return _expand_path(value) if value else None -SecretsKey = Annotated[str, BeforeValidator(validate_keys)] +SecretsKey = Annotated[SecretStr, BeforeValidator(validate_keys)] ExpandedPath = Annotated[Path, BeforeValidator(_expand_path)] OptionalExpandedPath = Annotated[Path | None, BeforeValidator(_expand_optional_path)] @@ -54,14 +65,89 @@ def _config_path() -> Path | None: return default if default.is_file() else None -class _PrunedTomlSource(TomlConfigSettingsSource): +def _secrets_dir() -> Path | None: + if configured := os.environ.get("DRUKS_SECRETS_DIR"): + path = Path(configured).expanduser() + if path.is_dir(): + return path + raise ValueError(f"DRUKS_SECRETS_DIR is not a directory: {path}") + + +def secret_variable(key: str) -> str: + """The environment variable of the secret ``key``: DRUKS_ and the key path, with + ``_`` for each dot.""" + return f"DRUKS_{key.replace('.', '_').upper()}" + + +def _pick(values: dict[str, Any], keys: tuple[str, ...]) -> dict[str, Any]: + """The dotted ``keys`` that ``values`` sets, in the nested shape of ``values``.""" + picked: dict[str, Any] = {} + for key in keys: + table, _, name = key.rpartition(".") + scope = values.get(table) if table else values + if isinstance(scope, dict) and name in scope: + target = picked.setdefault(table, {}) if table else picked + target[name] = scope[name] + return picked + + +class _TomlSource(TomlConfigSettingsSource): def __call__(self) -> dict[str, Any]: def drop_blank_values(value: Any) -> Any: if isinstance(value, dict): return {key: drop_blank_values(item) for key, item in value.items() if item != ""} return value - return drop_blank_values(super().__call__()) + document = drop_blank_values(super().__call__()) + for key in Settings.secret_keys(): + if _pick(document, (key,)): + raise ValueError( + f"druks.toml: {key} is a secret. Set {secret_variable(key)} in the " + f"environment, or write the file {key} in DRUKS_SECRETS_DIR." + ) + for name, field in Settings.model_fields.items(): + if field.alias and (name in document or field.alias in document): + raise ValueError( + f"druks.toml: {name} is not a druks.toml key. " + f"Set {field.alias} in the environment." + ) + return document + + +def _pick_injected(values: dict[str, Any], variables: Mapping[str, str | None]) -> dict[str, Any]: + """What the environment sets: the settings that carry an alias, and each secret + under the name that ``secret_variable`` builds from its key. A secret in both + places is refused, so neither one overrides.""" + aliases = {field.alias for field in Settings.model_fields.values() if field.alias} + secrets: dict[str, Any] = {} + secrets_dir = _secrets_dir() + for key in Settings.secret_keys(): + # The sources hold each variable name in lowercase. + variable = secret_variable(key).lower() + if variable in variables: + if secrets_dir and (secrets_dir / key).is_file(): + raise ValueError( + f"{key} is set in the environment and in a secret file. Remove one of them." + ) + table, _, name = key.rpartition(".") + target = secrets.setdefault(table, {}) if table else secrets + target[name] = variables[variable] + return {key: value for key, value in values.items() if key in aliases} | secrets + + +class _EnvSource(EnvSettingsSource): + def __call__(self) -> dict[str, Any]: + return _pick_injected(super().__call__(), self.env_vars) + + +class _DotEnvSource(DotEnvSettingsSource): + def __call__(self) -> dict[str, Any]: + return _pick_injected(super().__call__(), self.env_vars) + + +class _SecretsSource(NestedSecretsSettingsSource): + def __call__(self) -> dict[str, Any]: + return _pick(super().__call__(), Settings.secret_keys()) class Identity(BaseModel): @@ -121,16 +207,10 @@ def webhook_base(self) -> str: return self.endpoint.rstrip("/") -class Secrets(BaseModel): - # Encrypts stored secrets at rest. A missing or malformed key refuses boot; - # `druks setup` generates one. - secrets_key: SecretsKey - - class Sandbox(BaseModel): # The drukbox control plane. An empty service_url turns sandbox execution off. service_url: str = "" - service_token: str = "" + service_token: SecretStr = SecretStr("") # Empty → drukbox decides. image: str = "" # The issuer base URL the secrets exchange dials: the web process on the @@ -141,7 +221,7 @@ class Sandbox(BaseModel): browser_sandbox_image: str = "ghcr.io/czpython/druks/browser:latest" # An HTTP proxy for the login window only, so the login leaves from another # IP. It may carry a user name and password. Empty keeps the box IP. - browser_login_proxy: str = "" + browser_login_proxy: SecretStr = SecretStr("") # The IANA timezone of the login window, in the login proxy's region. Empty # keeps the container default. browser_login_tz: str = "" @@ -152,6 +232,7 @@ class Sandbox(BaseModel): class Settings(BaseSettings): model_config = SettingsConfigDict( populate_by_name=True, + env_prefix="DRUKS_", env_file=".env", env_file_encoding="utf-8", extra="ignore", @@ -166,24 +247,25 @@ class Settings(BaseSettings): timezone: Annotated[str, AfterValidator(validate_timezone)] = "UTC" identity: Identity = Identity() urls: Urls = Urls() - secrets: Secrets sandbox: Sandbox = Sandbox() + # Encrypts stored secrets at rest. A missing or malformed key refuses boot; + # `druks setup` generates one. + secrets_key: SecretsKey + # ``data_dir`` is the root for files, run artifacts, and logs (via computed # properties below). data_dir: ExpandedPath = Field(default=DEFAULT_DATA_DIR, alias="DRUKS_DATA_DIR") # Postgres connection URL. Every engine factory and Alembic read this. - database_url: str = Field( - default="postgresql+psycopg://druks:druks@localhost:5432/druks", - alias="DRUKS_DATABASE_URL", - ) + database_url: SecretStr = SecretStr("postgresql+psycopg://druks:druks@localhost:5432/druks") # SQLAlchemy reads a pool size of 0 as unbounded. database_pool_size: int = Field(default=20, gt=0, alias="DRUKS_DATABASE_POOL_SIZE") database_max_overflow: int = Field(default=30, ge=0, alias="DRUKS_DATABASE_MAX_OVERFLOW") dbos_pool_size: int = Field(default=20, gt=0, alias="DRUKS_DBOS_POOL_SIZE") - redis_url: str = Field(default="redis://127.0.0.1:6379/0", alias="DRUKS_REDIS_URL") + # A secret, because the URL can carry the Redis password. + redis_url: SecretStr = SecretStr("redis://127.0.0.1:6379/0") # Per-VM SSH keys when drukbox returns them; empty otherwise. sandbox_keys_dir: ExpandedPath = Field( default=DEFAULT_DATA_DIR / "sandbox-keys", @@ -225,13 +307,37 @@ def settings_customise_sources( dotenv_settings: PydanticBaseSettingsSource, file_secret_settings: PydanticBaseSettingsSource, ) -> tuple[PydanticBaseSettingsSource, ...]: + """Each setting has one source. druks.toml sets ``timezone`` and the tables. The + environment sets the settings that carry an alias. A secret comes from the + environment, or from its file when DRUKS_SECRETS_DIR names a directory.""" return ( init_settings, - _PrunedTomlSource(settings_cls, toml_file=_config_path()), - env_settings, - dotenv_settings, + _TomlSource(settings_cls, toml_file=_config_path()), + _EnvSource(settings_cls), + _DotEnvSource(settings_cls), + _SecretsSource( + file_secret_settings, + secrets_dir=_secrets_dir(), + secrets_nested_delimiter=".", + secrets_prefix="", + ), ) + @classmethod + def secret_keys(cls) -> tuple[str, ...]: + """The dotted key of each secret: its file name in DRUKS_SECRETS_DIR.""" + keys: list[str] = [] + for name, field in cls.model_fields.items(): + if field.annotation is SecretStr: + keys.append(name) + elif isinstance(field.annotation, type) and issubclass(field.annotation, BaseModel): + keys.extend( + f"{name}.{key}" + for key, table_field in field.annotation.model_fields.items() + if table_field.annotation is SecretStr + ) + return tuple(keys) + @property def logs_dir(self) -> Path: return self.data_dir / "logs" diff --git a/backend/druks/setup_env.py b/backend/druks/setup_env.py index 21c41d383..5144da5be 100644 --- a/backend/druks/setup_env.py +++ b/backend/druks/setup_env.py @@ -12,6 +12,7 @@ import tomlkit from druks.core.utils.time import validate_timezone +from druks.settings import Settings, secret_variable GAPS_EXIT_CODE = 3 @@ -27,8 +28,27 @@ ) _ENV_KEY_PATTERN = re.compile(r"[A-Za-z_][A-Za-z0-9_]*") -# Env keys druks owns — setup renders them into .env, or the app reads their -# value from druks.toml directly. [env] and provider tables may not carry them. +# The variables of each secret that setup knows, by its druks.toml key. Setup moves +# such a secret from druks.toml to the secrets section of .env. The secrets.* keys +# and the env.* keys cover a druks.toml from before that section. +_SECRET_VARIABLES = { + **{key: (secret_variable(key),) for key in Settings.secret_keys()}, + "sandbox.service_token": (secret_variable("sandbox.service_token"), "SERVICE_TOKENS"), + "sandbox.registry_password": ("REGISTRY_PASSWORD",), + "sandbox.exe.EXE_API_TOKEN": ("EXE_API_TOKEN",), + "sandbox.exe.TAILSCALE_OAUTH_CLIENT_SECRET": ("TAILSCALE_OAUTH_CLIENT_SECRET",), + "secrets.secrets_key": ("DRUKS_SECRETS_KEY",), + "secrets.postgres_password": ("DRUKS_POSTGRES_PASSWORD",), + "secrets.drukbox_secrets_key": ("SECRETS_KEY",), + "env.DRUKS_DATABASE_URL": ("DRUKS_DATABASE_URL",), + "env.DRUKS_REDIS_URL": ("DRUKS_REDIS_URL",), +} +_KNOWN_SECRETS = frozenset(name for names in _SECRET_VARIABLES.values() for name in names) +_SECRETS_TITLE = "SECRETS" + +# Env keys druks owns — setup renders them into .env, the app reads their value +# from druks.toml directly, or they are known secrets. [env] and provider tables +# may not carry them. _OWNED_ENV_KEYS = frozenset( { "DRUKS_POSTGRES_PASSWORD", @@ -65,6 +85,7 @@ "SECRETS_EXCHANGE_PORT", "DRUKS_SECRETS_PROXY_BIND_HOST", } + | _KNOWN_SECRETS ) _KNOWN_TOP_LEVEL_KEYS = frozenset({"timezone"}) _KNOWN_TOML_KEYS = { @@ -77,24 +98,16 @@ "jwt_identity_claim", ), "urls": ("endpoint", "webhook_host"), - "secrets": ( - "postgres_password", - "secrets_key", - "drukbox_secrets_key", - ), "paths": ("data_dir", "harness_config_root"), "sandbox": ( "provider", "service_url", - "service_token", "image", "registry_host", "registry_username", - "registry_password", "template_repository", "proxy_url", "issuer_url", - "browser_login_proxy", "browser_login_tz", "timeout", ), @@ -111,12 +124,14 @@ def _secrets_key() -> str: def read_env(path: Path) -> dict[str, str]: + return _parse_env(path.read_text()) if path.exists() else {} + + +def _parse_env(text: str) -> dict[str, str]: values: dict[str, str] = {} - if path.exists(): - for line in path.read_text().splitlines(): - if not line or line.startswith("#") or "=" not in line: - continue - key, _, value = line.partition("=") + for line in text.splitlines(): + key, separator, value = line.partition("=") + if separator and not line.startswith("#"): values[key.strip()] = value return values @@ -147,24 +162,32 @@ def run_setup( _set_value(document, value_path, value) is_changed = True + existing_env = env_path.read_text() if env_path.exists() else "" + secrets = _read_secrets(existing_env) + is_changed = _move_secrets(document, secrets) or is_changed + _generate_secrets(secrets) + provider = _get_string(document, ("sandbox", "provider")) print_fn(_shape_message(provider)) + toml_text = tomlkit.dumps(document) + config = _canonical_config(tomllib.loads(toml_text)) + extras = {key: value for key, value in read_env(env_path).items() if key in _COMPOSE_ENV_KEYS} + env_text = _render_env(config, extras=extras, secrets=secrets) + # .env holds the secrets. Replace it in one step, and before druks.toml loses + # a secret that this run moves. + partial_env_path = env_path.with_name(f"{env_path.name}.tmp") + _write_secure_text(partial_env_path, env_text) + os.replace(partial_env_path, env_path) if is_changed: - _write_toml(toml_path, document) - config = _read_toml(toml_path) + _write_secure_text(toml_path, toml_text) if is_fresh: _write_gitignore(env_path.parent / ".gitignore") - existing_env = env_path.read_text() if env_path.exists() else "" - extras = {key: value for key, value in read_env(env_path).items() if key in _COMPOSE_ENV_KEYS} - env_text = _render_env(config, extras=extras) - _write_secure_text(env_path, env_text) - if existing_env and existing_env != env_text: print_fn("Rendered .env changed. Apply it with: docker compose up -d") - gaps = _collect_gaps(config) + gaps = _collect_gaps(config, secrets) _print_outcome(print_fn, env_path=env_path, provider=provider, gaps=gaps) return GAPS_EXIT_CODE if gaps else 0 @@ -172,7 +195,8 @@ def run_setup( _TOML_TEMPLATE = """\ # druks.toml — the deployment. Edit this file, then re-run the installer # to render and apply it. `druks setup` alone re-renders .env but does -# not restart services. +# not restart services. This file holds no secret: the secrets are in the +# last section of .env. See configuration.md. # Schedule timezone and initial timezone for new accounts. timezone = "UTC" @@ -191,12 +215,6 @@ def run_setup( endpoint = "" webhook_host = "" -# Generated on first write. Do not regenerate a deployed secret. -[secrets] -postgres_password = "" -secrets_key = "" -drukbox_secrets_key = "" - # Host paths. [paths] data_dir = "" @@ -206,12 +224,11 @@ def run_setup( [sandbox] provider = "" service_url = "" -service_token = "" image = "" # Access to private sandbox images on one registry host, for example ghcr.io. +# The password is the secret REGISTRY_PASSWORD in .env. registry_host = "" registry_username = "" -registry_password = "" # The repository path on that host where drukbox publishes sandbox templates. # The exe provider requires it. template_repository = "" @@ -223,20 +240,16 @@ def run_setup( # The issuer base URL the secrets exchange dials; loopback web by default. For a # drukbox on another server, set the address of this host that drukbox reaches. issuer_url = "" -# An HTTP proxy for the login window. The login then leaves from a different IP -# than the box. Use it for sign-in flows that refuse the box IP. Examples: -# http://172.17.0.1:8888, or http://user:pass@host:port for a proxy with a user -# name and password. If it is empty, the login uses the box IP. Only the login -# window uses it. See configuration.md. -browser_login_proxy = "" # The timezone of the login browser. Use an IANA zone, for example -# "Europe/Madrid". Set it to the region of the login proxy. If it is empty, the -# browser keeps the container default. +# "Europe/Madrid". Set it to the region of the login proxy, which is the secret +# DRUKS_SANDBOX_BROWSER_LOGIN_PROXY in .env. If it is empty, the browser keeps +# the container default. browser_login_tz = "" timeout = 180 # Put drukbox environment in [sandbox.]. The table is passed through # to remote stacks verbatim; the local docker shape renders no provider table. +# Put a provider secret in the secrets section of .env, not in this table. # Provider reference: https://github.com/czpython/drukbox (docs/deploy.md). # Raw environment for processes druks does not model (drukbox, Caddy, libraries @@ -253,7 +266,6 @@ def _fresh_values(*, provider: str, home: str) -> tuple[tuple[tuple[str, ...], s # matches the origin every local doc prints. (("urls", "endpoint"), "http://127.0.0.1:8001"), (("sandbox", "service_url"), "http://127.0.0.1:8780"), - (("sandbox", "service_token"), "dev-token"), (("sandbox", "image"), "ghcr.io/czpython/druks/sandbox:latest"), # Sandbox containers reach the host at the bridge gateway. (("sandbox", "proxy_url"), "http://172.17.0.1:8880"), @@ -263,11 +275,8 @@ def _fresh_values(*, provider: str, home: str) -> tuple[tuple[tuple[str, ...], s (("identity", "mode"), "header"), (("identity", "header"), "X-ExeDev-Email"), (("sandbox", "service_url"), "http://127.0.0.1:8780"), - (("sandbox", "service_token"), _hex_secret()), - (("sandbox", "exe", "EXE_API_TOKEN"), ""), (("sandbox", "exe", "TAILSCALE_TAILNET"), ""), (("sandbox", "exe", "TAILSCALE_OAUTH_CLIENT_ID"), ""), - (("sandbox", "exe", "TAILSCALE_OAUTH_CLIENT_SECRET"), ""), (("sandbox", "exe", "EXE_API_URL"), "https://exe.dev"), (("sandbox", "exe", "EXE_DEFAULT_IMAGE"), "ghcr.io/boldsoftware/exeuntu:latest"), (("sandbox", "exe", "TAILSCALE_ENABLED"), "true"), @@ -278,26 +287,16 @@ def _fresh_values(*, provider: str, home: str) -> tuple[tuple[tuple[str, ...], s shape = ( (("identity", "mode"), "header"), (("sandbox", "service_url"), "http://127.0.0.1:8780"), - (("sandbox", "service_token"), _hex_secret()), ) return ( (("sandbox", "provider"), provider), - (("secrets", "postgres_password"), _hex_secret()), - (("secrets", "secrets_key"), _secrets_key()), - (("secrets", "drukbox_secrets_key"), _secrets_key()), (("paths", "data_dir"), f"{home.rstrip('/')}/druks-data"), (("paths", "harness_config_root"), f"{home.rstrip('/')}/.config/druks/harnesses"), *shape, ) -def _read_toml(path: Path) -> dict[str, Any]: - with path.open("rb") as config_file: - config = tomllib.load(config_file) - return _canonical_config(config) - - def _canonical_config(raw: dict[str, Any]) -> dict[str, Any]: """Validated copy with every known table and key present. Operator additions are flat scalars, one table deep; more structure is refused by key.""" @@ -377,6 +376,47 @@ def _set_value(target: MutableMapping[str, Any], path: tuple[str, ...], value: s current[path[-1]] = value +def _read_secrets(env_text: str) -> dict[str, str]: + """The secrets that .env holds: each variable of its secrets section, and each + known secret above that section.""" + head, _, section = env_text.partition(f"\n# {_SECRETS_TITLE}\n") + secrets = {key: value for key, value in _parse_env(head).items() if key in _KNOWN_SECRETS} + for key, value in _parse_env(section).items(): + # install.sh appends its compose keys to the end of the file. + if key not in _COMPOSE_ENV_KEYS: + secrets[key] = value + return {key: value for key, value in secrets.items() if value} + + +def _move_secrets(document: tomlkit.TOMLDocument, secrets: dict[str, str]) -> bool: + """Move each known secret that druks.toml holds to ``secrets``.""" + is_moved = False + for key, variables in _SECRET_VARIABLES.items(): + *table_path, name = key.split(".") + table: Any = document + for part in table_path: + table = table.get(part, {}) if isinstance(table, MutableMapping) else {} + if isinstance(table, MutableMapping) and name in table: + if value := _get_string(table, (name,)): + secrets.update(dict.fromkeys(variables, value)) + del table[name] + is_moved = True + if "secrets" in document and not document["secrets"]: + del document["secrets"] + return is_moved + + +def _generate_secrets(secrets: dict[str, str]) -> None: + """Add each secret that setup can make and that ``secrets`` does not hold.""" + secrets.setdefault("DRUKS_SECRETS_KEY", _secrets_key()) + secrets.setdefault("SECRETS_KEY", _secrets_key()) + secrets.setdefault("DRUKS_POSTGRES_PASSWORD", _hex_secret()) + token = secrets.setdefault(secret_variable("sandbox.service_token"), _hex_secret()) + # drukbox refuses to start without SERVICE_TOKENS. A compose-side default + # would replace that safe stop with a known token. + secrets.setdefault("SERVICE_TOKENS", token) + + def _parse_assignment(assignment: str) -> tuple[tuple[str, ...], str]: path_text, separator, value = assignment.partition("=") path = tuple(path_text.split(".")) @@ -389,30 +429,17 @@ def _parse_assignment(assignment: str) -> tuple[tuple[str, ...], str]: return path, value -def _write_toml(path: Path, document: tomlkit.TOMLDocument) -> None: - text = tomlkit.dumps(document) - _canonical_config(tomllib.loads(text)) - _write_secure_text(path, text) - - def _render_env( config: dict[str, Any], *, extras: dict[str, str], + secrets: dict[str, str], ) -> str: provider = _get_string(config, ("sandbox", "provider")) - - # drukbox refuses to start without SERVICE_TOKENS. A compose-side default - # would replace that safe stop with a known token. - service_tokens = _get_string(config, ("sandbox", "service_token")) proxy_url = _get_string(config, ("sandbox", "proxy_url")) issuer_url = _get_string(config, ("sandbox", "issuer_url")) sections = ( - ( - "GENERATED SECRETS", - (("DRUKS_POSTGRES_PASSWORD", _get_string(config, ("secrets", "postgres_password"))),), - ), ( "DEPLOYMENT DEFAULTS", ( @@ -435,12 +462,9 @@ def _render_env( "SANDBOX", ( ("DEFAULT_HOST_PROVIDER", provider), - ("SERVICE_TOKENS", service_tokens), ("REGISTRY_HOST", _get_string(config, ("sandbox", "registry_host"))), ("REGISTRY_USERNAME", _get_string(config, ("sandbox", "registry_username"))), - ("REGISTRY_PASSWORD", _get_string(config, ("sandbox", "registry_password"))), ("TEMPLATE_REPOSITORY", _get_string(config, ("sandbox", "template_repository"))), - ("SECRETS_KEY", _get_string(config, ("secrets", "drukbox_secrets_key"))), ("SECRETS_PROXY_URL", proxy_url), # The proxy binds the address sandboxes dial and nothing else. ("DRUKS_SECRETS_PROXY_BIND_HOST", urlsplit(proxy_url).hostname or ""), @@ -509,6 +533,19 @@ def _render_env( if key in compose_extras: lines.append(_env_line(key, compose_extras[key])) lines.append("") + + # The last section, because setup reads it back to the end of the file. + lines.extend( + ( + "# " + "=" * 60, + f"# {_SECRETS_TITLE}", + "# " + "=" * 60, + "# druks setup makes these values once and keeps each line of this section.", + "# Add your own secrets here, for example a provider token.", + "", + *(_env_line(key, value) for key, value in secrets.items()), + ) + ) return "\n".join(lines).rstrip() + "\n" @@ -522,17 +559,13 @@ def _is_reserved_env_key(key: str) -> bool: return key.startswith("DRUKS_") or key in _OWNED_ENV_KEYS -def _collect_gaps(config: dict[str, Any]) -> list[str]: +def _collect_gaps(config: dict[str, Any], secrets: dict[str, str]) -> list[str]: gaps = [ f"{'.'.join(path)} is empty" for path in ( - ("secrets", "postgres_password"), - ("secrets", "secrets_key"), - ("secrets", "drukbox_secrets_key"), ("identity", "mode"), ("sandbox", "provider"), ("sandbox", "service_url"), - ("sandbox", "service_token"), ) if not _get_string(config, path) ] @@ -567,14 +600,22 @@ def _collect_gaps(config: dict[str, Any]) -> list[str]: gaps.append(f"env.{key} is reserved by druks") provider_environment = provider_tables.get(provider, {}) + for key in sorted(secrets.keys() & (deployment_env.keys() | provider_environment.keys())): + gaps.append( + f"{key} is in druks.toml and in the secrets section of .env. Remove one of them." + ) + if provider == "exe": - for key in ("EXE_API_TOKEN", "TAILSCALE_TAILNET"): - if not _get_string(config, ("sandbox", "exe", key)): - gaps.append(f"sandbox.exe.{key} is empty") + if "EXE_API_TOKEN" not in secrets: + gaps.append("EXE_API_TOKEN is empty. Add it to the secrets section of .env.") + if not _get_string(config, ("sandbox", "exe", "TAILSCALE_TAILNET")): + gaps.append("sandbox.exe.TAILSCALE_TAILNET is empty") if not _get_string(config, ("sandbox", "proxy_url")): gaps.append("sandbox.proxy_url is empty") elif provider != "docker" and not any( - value for key, value in provider_environment.items() if not _is_reserved_env_key(key) + value + for key, value in (provider_environment | secrets).items() + if not _is_reserved_env_key(key) ): gaps.append( f"[sandbox.{provider}] has no configured values for remote provider {provider!r}" @@ -627,6 +668,6 @@ def _print_outcome( for gap in gaps: print_fn(f" - {gap}") print_fn("") - print_fn("Set the values in druks.toml, then re-run the installer.") + print_fn("Set the values in druks.toml and .env, then re-run the installer.") else: print_fn(f"✓ {env_path} is complete (provider: {provider}).") diff --git a/backend/druks/testing.py b/backend/druks/testing.py index cd2e1c816..253462cdc 100644 --- a/backend/druks/testing.py +++ b/backend/druks/testing.py @@ -118,17 +118,17 @@ def pytest_configure(config) -> None: # import an app's workflows module before every installed package is claimed. iter_apps() - with tempfile.NamedTemporaryFile(mode="w", suffix=".toml", delete=False) as settings_file: - settings_file.write( - f'[secrets]\nsecrets_key = "{base64.b64encode(secrets.token_bytes(32)).decode()}"\n' - ) - settings_path = Path(settings_file.name) + # An empty file, so the app under test never reads a druks.toml from the + # working directory. + with tempfile.NamedTemporaryFile(suffix=".toml", delete=False) as settings_file: + settings_path = Path(settings_file.name) os.environ.setdefault("DRUKS_CONFIG", str(settings_path)) config.add_cleanup(settings_path.unlink) # Under pytest, druks IS the test instance. Infrastructure settings are read from # the environment wherever they're needed — the app's Redis dialer among them — so # overriding here keeps the code under test on the database and Redis the fixtures # own. + os.environ.setdefault("DRUKS_SECRETS_KEY", base64.b64encode(secrets.token_bytes(32)).decode()) os.environ["DRUKS_DATABASE_URL"] = TEST_DATABASE_URL os.environ["DRUKS_REDIS_URL"] = TEST_REDIS_URL # App commits become savepoints inside the fixture's outer transaction, which @@ -216,9 +216,7 @@ def make_settings(tmp_path: Path, **overrides: object) -> Settings: defaults = { "data_dir": tmp_path, "database_url": TEST_DATABASE_URL, - "secrets": { - "secrets_key": "MDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDA=", - }, + "secrets_key": "MDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDA=", "redis_url": TEST_REDIS_URL, "log_level": "WARNING", } diff --git a/backend/tests/test_browser_sessions.py b/backend/tests/test_browser_sessions.py index 06e52af17..26cd4ce36 100644 --- a/backend/tests/test_browser_sessions.py +++ b/backend/tests/test_browser_sessions.py @@ -147,7 +147,7 @@ async def test_import_materializes_the_row_survives_restart_and_delete_removes_i assert row.payload.decrypt() == payload wrong_key = base64.b64encode(b"1" * 32).decode() - wrong_settings = make_settings(tmp_path / "wrong", secrets={"secrets_key": wrong_key}) + wrong_settings = make_settings(tmp_path / "wrong", secrets_key=wrong_key) with monkeypatch.context() as patch: # The plane reads its keys through druks.database at each use. patch.setattr(database, "load_settings", lambda: wrong_settings) diff --git a/backend/tests/test_secrets.py b/backend/tests/test_secrets.py index 8df923e44..c025130ae 100644 --- a/backend/tests/test_secrets.py +++ b/backend/tests/test_secrets.py @@ -20,10 +20,8 @@ def _key() -> str: return base64.b64encode(os.urandom(32)).decode() -def _set_key(monkeypatch, tmp_path, value: str) -> None: - config_path = tmp_path / "druks.toml" - config_path.write_text(f'[secrets]\nsecrets_key = "{value}"\n') - monkeypatch.setenv("DRUKS_CONFIG", str(config_path)) +def _set_key(monkeypatch, value: str) -> None: + monkeypatch.setenv("DRUKS_SECRETS_KEY", value) async def _store_token(token: str = _TOKEN) -> None: @@ -82,52 +80,52 @@ async def test_grant_secret_halves_round_trip(druks_db): assert grant.secrets["client_secret"] == "cs-secret" -async def test_loaded_secrets_are_lazy_and_redacted(monkeypatch, tmp_path, druks_db): +async def test_loaded_secrets_are_lazy_and_redacted(monkeypatch, druks_db): await _store_token() druks_db.expunge_all() # Loading and logging a row never touches key material — decryption # happens only on a read of a value, and repr leaks nothing either way. row = await _get_token() - _set_key(monkeypatch, tmp_path, "") + _set_key(monkeypatch, "") assert repr(row.secrets) == "SecretsMapping()" - with pytest.raises(ValidationError, match="Field required"): + with pytest.raises(ValidationError, match="at least one"): row.secrets["value"] -def test_missing_key_refuses_boot(monkeypatch, tmp_path): +def test_missing_key_refuses_boot(monkeypatch): # Blank and comma-noise-only both read as "no key" — the required setting # refuses at construction rather than falling back to plaintext. for broken in ("", ",", " , "): - _set_key(monkeypatch, tmp_path, broken) - with pytest.raises(ValidationError, match="Field required|at least one"): + _set_key(monkeypatch, broken) + with pytest.raises(ValidationError, match="at least one"): load_settings() -def test_key_validation_error_never_echoes_the_key(monkeypatch, tmp_path): +def test_key_validation_error_never_echoes_the_key(monkeypatch): # A half-valid list fails validation, and the failure surfaces in boot # logs and doctor output — it must not echo the valid segment. good = _key() - _set_key(monkeypatch, tmp_path, f"{good},not-base64!!") + _set_key(monkeypatch, f"{good},not-base64!!") with pytest.raises(ValidationError) as error_info: load_settings() assert good not in str(error_info.value) -def test_malformed_key_refuses_boot(monkeypatch, tmp_path): +def test_malformed_key_refuses_boot(monkeypatch): for broken in ("not-base64!!", base64.b64encode(b"short").decode()): - _set_key(monkeypatch, tmp_path, broken) + _set_key(monkeypatch, broken) with pytest.raises(ValidationError, match="base64|32 bytes"): load_settings() -async def test_undecryptable_secret_raises_the_named_error(monkeypatch, tmp_path, druks_db): +async def test_undecryptable_secret_raises_the_named_error(monkeypatch, druks_db): # A key dropped from the list while rows written under it existed is the # usual cause — the error must say so, not surface a bare crypto traceback. await _store_token() druks_db.expunge_all() - _set_key(monkeypatch, tmp_path, _key()) + _set_key(monkeypatch, _key()) with pytest.raises(SecretDecryptError, match="rotated out"): (await _get_token()).secrets["value"] @@ -145,15 +143,15 @@ async def test_garbled_envelope_raises_the_named_error(druks_db): (await _get_token()).secrets["value"] -async def test_prepended_key_still_decrypts(monkeypatch, tmp_path, druks_db): +async def test_prepended_key_still_decrypts(monkeypatch, druks_db): # Rotation is prepend-only: new writes use the first key; rows written # under an older key keep decrypting as long as it stays in the list. old_key = _key() - _set_key(monkeypatch, tmp_path, old_key) + _set_key(monkeypatch, old_key) await _store_token() await _store_grant(refresh_token="rt-secret") - _set_key(monkeypatch, tmp_path, f"{_key()},{old_key}") + _set_key(monkeypatch, f"{_key()},{old_key}") druks_db.expunge_all() assert (await _get_token()).secrets["value"] == _TOKEN [grant] = await VaultSecret.list_connections(druks_db, Audience.mcp("notion")) diff --git a/backend/tests/test_settings.py b/backend/tests/test_settings.py index 0a1928e93..fe662bdbd 100644 --- a/backend/tests/test_settings.py +++ b/backend/tests/test_settings.py @@ -5,7 +5,7 @@ from druks import database from druks.durable import engine as durable_engine from druks.settings import Settings, ensure_data_dirs -from druks.testing import make_settings +from druks.testing import TEST_DATABASE_URL, make_settings from pydantic import ValidationError _SECRETS_KEY = "MDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDA=" @@ -60,10 +60,21 @@ def test_the_pat_slot_cannot_be_the_identity_header(tmp_path, mode): ) -def test_toml_populates_authored_submodels(tmp_path, monkeypatch): +def _write_secrets(tmp_path, monkeypatch, secrets): + """Give Druks its secrets as files, and no secret in the environment.""" + secrets_dir = tmp_path / "secrets" + secrets_dir.mkdir() + for name, value in {"secrets_key": _SECRETS_KEY, **secrets}.items(): + (secrets_dir / name).write_text(value) + monkeypatch.setenv("DRUKS_SECRETS_DIR", str(secrets_dir)) + for variable in ("DRUKS_SECRETS_KEY", "DRUKS_DATABASE_URL", "DRUKS_REDIS_URL"): + monkeypatch.delenv(variable) + + +def test_toml_sets_the_tables_and_the_secrets_directory_sets_the_secrets(tmp_path, monkeypatch): config_path = tmp_path / "druks.toml" config_path.write_text( - f''' + """ timezone = "Europe/Madrid" [identity] mode = "header" @@ -73,18 +84,23 @@ def test_toml_populates_authored_submodels(tmp_path, monkeypatch): endpoint = "https://druks.example.com" webhook_host = "hooks.example.com" -[secrets] -secrets_key = "{_SECRETS_KEY}" - [sandbox] service_url = "https://sandbox.example.com" -service_token = "sandbox-token" image = "sandbox:latest" timeout = 180 -'''.strip() +""".strip() + "\n" ) monkeypatch.setenv("DRUKS_CONFIG", str(config_path)) + _write_secrets( + tmp_path, + monkeypatch, + { + "database_url": "postgresql+psycopg://druks:database-password@db/druks", + "redis_url": "redis://:redis-password@redis:6379/3", + "sandbox.service_token": "sandbox-token", + }, + ) settings = Settings() @@ -92,12 +108,57 @@ def test_toml_populates_authored_submodels(tmp_path, monkeypatch): assert settings.identity.header == "X-Edge-Email" assert settings.urls.endpoint == "https://druks.example.com" assert settings.urls.webhook_host == "hooks.example.com" - assert settings.secrets.secrets_key == _SECRETS_KEY - assert settings.sandbox.service_token == "sandbox-token" + assert settings.secrets_key.get_secret_value() == _SECRETS_KEY + assert settings.sandbox.service_token.get_secret_value() == "sandbox-token" + assert settings.redis_url.get_secret_value() == "redis://:redis-password@redis:6379/3" assert settings.sandbox.service_url == "https://sandbox.example.com" assert settings.sandbox.image == "sandbox:latest" assert settings.sandbox.timeout == 180.0 assert settings.timezone == "Europe/Madrid" + for secret in (_SECRETS_KEY, "database-password", "redis-password", "sandbox-token"): + assert secret not in repr(settings) + + +@pytest.mark.parametrize( + ("body", "message"), + [ + ('secrets_key = "key"', "secrets_key is a secret"), + ('[sandbox]\nservice_token = "token"', "sandbox.service_token is a secret"), + ('data_dir = "/home/op/druks-data"', "data_dir is not a druks.toml key"), + ], +) +def test_druks_toml_refuses_a_secret_and_a_setting_of_the_environment( + tmp_path, monkeypatch, body, message +): + config_path = tmp_path / "druks.toml" + config_path.write_text(body + "\n") + monkeypatch.setenv("DRUKS_CONFIG", str(config_path)) + + with pytest.raises(ValueError, match=message): + Settings() + + +def test_the_environment_sets_a_secret_under_its_prefixed_name(tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + monkeypatch.setenv("DRUKS_SANDBOX_SERVICE_TOKEN", "environment-token") + monkeypatch.setenv("DRUKS_TIMEZONE", "Asia/Tokyo") + # Drukbox reads these names, and they share the environment file of an install. + monkeypatch.setenv("DATABASE_URL", "sqlite:///drukbox") + monkeypatch.setenv("SECRETS_KEY", "drukbox-key") + + settings = Settings() + + assert settings.sandbox.service_token.get_secret_value() == "environment-token" + assert settings.database_url.get_secret_value() == TEST_DATABASE_URL + assert settings.timezone == "UTC" + + +def test_a_secret_in_the_environment_and_in_a_file_refuses_construction(tmp_path, monkeypatch): + _write_secrets(tmp_path, monkeypatch, {}) + monkeypatch.setenv("DRUKS_SECRETS_KEY", _SECRETS_KEY) + + with pytest.raises(ValueError, match="secrets_key is set in the environment and in a secret"): + Settings() def test_only_an_explicit_issuer_url_changes_the_mint_base(tmp_path): @@ -114,23 +175,14 @@ def test_auth_mode_environment_variable_is_ignored(tmp_path, monkeypatch): monkeypatch.delenv("DRUKS_CONFIG", raising=False) monkeypatch.setenv("DRUKS_AUTH_MODE", "header") - settings = Settings(secrets={"secrets_key": _SECRETS_KEY}) + settings = Settings() assert settings.identity.mode == "none" def test_blank_toml_value_uses_submodel_default(tmp_path, monkeypatch): config_path = tmp_path / "druks.toml" - config_path.write_text( - f""" -[identity] -jwt_identity_claim = "" - -[secrets] -secrets_key = "{_SECRETS_KEY}" -""".strip() - + "\n" - ) + config_path.write_text('[identity]\njwt_identity_claim = ""\n') monkeypatch.setenv("DRUKS_CONFIG", str(config_path)) settings = Settings() @@ -184,8 +236,8 @@ def test_development_example_pins_the_installation_timezone(tmp_path, monkeypatc config = tmp_path / "druks.toml" config.write_text(example.read_text()) monkeypatch.setenv("DRUKS_CONFIG", str(config)) - monkeypatch.setenv("TIMEZONE", "Asia/Tokyo") - assert Settings(secrets={"secrets_key": _SECRETS_KEY}).timezone == "UTC" + monkeypatch.setenv("DRUKS_TIMEZONE", "Asia/Tokyo") + assert Settings().timezone == "UTC" def test_pool_settings_reach_the_app_engine_and_dbos(tmp_path, monkeypatch): @@ -198,7 +250,7 @@ def test_pool_settings_reach_the_app_engine_and_dbos(tmp_path, monkeypatch): configs = [] monkeypatch.setattr(durable_engine, "DBOS", lambda config: configs.append(config)) - pool = database.create_async_engine_from_url(settings.database_url).pool + pool = database.create_async_engine_from_url(settings.database_url.get_secret_value()).pool durable_engine.init_dbos() assert (pool.size(), pool._max_overflow) == (3, 4) diff --git a/backend/tests/test_setup_env.py b/backend/tests/test_setup_env.py index ab6a2fe01..cdbdafe65 100644 --- a/backend/tests/test_setup_env.py +++ b/backend/tests/test_setup_env.py @@ -13,9 +13,7 @@ "DRUKS_AUTH_JWT_AUDIENCE": ("identity.jwt_audience", "druks"), "DRUKS_AUTH_JWT_IDENTITY_CLAIM": ("identity.jwt_identity_claim", "/sub"), "DRUKS_ENDPOINT": ("urls.endpoint", "https://druks.example"), - "DRUKS_SECRETS_KEY": ("secrets.secrets_key", "secrets-key"), "DRUKS_SANDBOX_SERVICE_URL": ("sandbox.service_url", "http://sandbox:8000"), - "DRUKS_SANDBOX_SERVICE_TOKEN": ("sandbox.service_token", "token"), "DRUKS_SANDBOX_IMAGE": ("sandbox.image", "sandbox:latest"), } @@ -35,8 +33,9 @@ def _read_toml(path: Path) -> dict: return tomllib.load(config_file) -def drukbox_key(tmp_path: Path) -> str: - return _read_toml(tmp_path / "druks.toml")["secrets"]["drukbox_secrets_key"] +def _read_secrets(env_path: Path) -> str: + """The secrets section of .env.""" + return env_path.read_text().partition("# SECRETS\n")[2] def test_fresh_exe_render_matches_the_deployment_contract(tmp_path): @@ -52,8 +51,6 @@ def test_fresh_exe_render_matches_the_deployment_contract(tmp_path): assert values["EXE_API_URL"] == "https://exe.dev" assert values["EXE_DEFAULT_IMAGE"] == "ghcr.io/boldsoftware/exeuntu:latest" assert values["DRUKS_AUTH_HEADER"] == "X-ExeDev-Email" - assert values["SERVICE_TOKENS"] == config["sandbox"]["service_token"] - assert values["SECRETS_KEY"] == config["secrets"]["drukbox_secrets_key"] assert "SECRETS_PROXY_URL" not in values assert "DRUKS_SECRETS_PROXY_BIND_HOST" not in values assert values["DRUKS_DATA_DIR"] == "/home/op/druks-data" @@ -61,9 +58,6 @@ def test_fresh_exe_render_matches_the_deployment_contract(tmp_path): assert config["paths"]["harness_config_root"] == values["DRUKS_HARNESS_CONFIG_ROOT"] assert "EXE_API_TOKEN" not in values assert "TAILSCALE_TAILNET" not in values - assert len(config["secrets"]["postgres_password"]) == 64 - assert len(config["secrets"]["drukbox_secrets_key"]) == 44 - assert len(config["sandbox"]["service_token"]) == 64 assert (tmp_path / ".gitignore").read_text().splitlines() == [ "druks.toml", ".env", @@ -72,6 +66,28 @@ def test_fresh_exe_render_matches_the_deployment_contract(tmp_path): assert stat.S_IMODE((tmp_path / "druks.toml").stat().st_mode) == 0o600 +def test_setup_generates_the_secrets_in_env_and_none_in_druks_toml(tmp_path): + env_path = tmp_path / ".env" + names = [ + "DRUKS_SECRETS_KEY", + "SECRETS_KEY", + "DRUKS_POSTGRES_PASSWORD", + "DRUKS_SANDBOX_SERVICE_TOKEN", + "SERVICE_TOKENS", + ] + + assert _run(env_path, provider="docker") == 0 + + values = read_env(env_path) + lines = [line for line in _read_secrets(env_path).splitlines() if line[:1] not in ("", "#")] + assert [line.partition("=")[0] for line in lines] == names + assert [len(values[name]) for name in names] == [44, 44, 64, 64, 64] + assert values["SERVICE_TOKENS"] == values["DRUKS_SANDBOX_SERVICE_TOKEN"] + toml_text = (tmp_path / "druks.toml").read_text() + for name in names: + assert values[name] not in toml_text + + def test_fresh_docker_run_is_boot_ready(tmp_path): env_path = tmp_path / ".env" printed = [] @@ -151,10 +167,6 @@ def test_docker_shape_matches_local_wiring_and_ignores_provider_environment(tmp_ values = read_env(env_path) assert values["DEFAULT_HOST_PROVIDER"] == "docker" assert "DRUKS_AUTH_HEADER" not in values - # Rendered on every shape. Without it, drukbox stops instead of falling - # back to a known token. - assert values["SERVICE_TOKENS"] == "dev-token" - assert values["SECRETS_KEY"] == drukbox_key(tmp_path) # Sandbox containers reach the host at the bridge gateway. The proxy binds # that address only. assert values["SECRETS_PROXY_URL"] == "http://172.17.0.1:8880" @@ -261,11 +273,11 @@ def test_set_updates_toml_and_rerender_preserves_the_values(tmp_path): def test_generated_secrets_never_regenerate_on_rerun(tmp_path): env_path = tmp_path / ".env" _run(env_path) - first = _read_toml(tmp_path / "druks.toml")["secrets"] + first = _read_secrets(env_path) _run(env_path, set_values=("urls.endpoint=https://druks.example",)) - assert _read_toml(tmp_path / "druks.toml")["secrets"] == first + assert _read_secrets(env_path) == first def test_deployment_env_addition_renders_verbatim_and_survives_rerender(tmp_path): @@ -332,15 +344,56 @@ def test_reserved_sandbox_env_key_is_a_named_gap_and_is_not_rendered(tmp_path): assert read_env(env_path)["DATABASE_URL"] == "sqlite+aiosqlite:////data/drukbox.db" -def test_deleted_env_is_regenerated_byte_identically(tmp_path): +def test_a_secret_that_the_operator_adds_to_env_survives_a_rerender(tmp_path): env_path = tmp_path / ".env" - _run(env_path) - expected = env_path.read_bytes() - env_path.unlink() + set_values = ("identity.header=X-Forwarded-Email",) + assert _run(env_path, provider="hetzner", set_values=set_values) == GAPS_EXIT_CODE + env_path.write_text(env_path.read_text() + "HETZNER_API_TOKEN=hetzner-token\n") + + assert _run(env_path) == 0 + + assert read_env(env_path)["HETZNER_API_TOKEN"] == "hetzner-token" + + +def test_a_variable_in_druks_toml_and_in_the_secrets_of_env_is_a_named_gap(tmp_path): + env_path = tmp_path / ".env" + printed = [] + _run(env_path, provider="exoscale", set_values=("identity.header=X-Forwarded-Email",)) + env_path.write_text(env_path.read_text() + "EXOSCALE_API_SECRET=exoscale-secret\n") + + rc = _run( + env_path, + set_values=("sandbox.exoscale.EXOSCALE_API_SECRET=another",), + print_fn=printed.append, + ) + + assert rc == GAPS_EXIT_CODE + gap = "EXOSCALE_API_SECRET is in druks.toml and in the secrets section of .env" + assert gap in "\n".join(printed) + + +def test_setup_moves_a_secret_that_druks_toml_holds_to_env(tmp_path): + env_path = tmp_path / ".env" + _run(env_path, set_values=("sandbox.exe.TAILSCALE_TAILNET=tail.ts.net",)) + toml_path = tmp_path / "druks.toml" + toml_path.write_text( + toml_path.read_text() + .replace("[paths]", '[secrets]\nsecrets_key = "old-vault-key"\n\n[paths]') + .replace("[sandbox]\n", '[sandbox]\nservice_token = "old-token"\n') + .replace("[sandbox.exe]\n", '[sandbox.exe]\nEXE_API_TOKEN = "old-exe-token"\n') + ) _run(env_path) - assert env_path.read_bytes() == expected + values = read_env(env_path) + config = _read_toml(toml_path) + assert values["DRUKS_SECRETS_KEY"] == "old-vault-key" + assert values["DRUKS_SANDBOX_SERVICE_TOKEN"] == "old-token" + assert values["SERVICE_TOKENS"] == "old-token" + assert values["EXE_API_TOKEN"] == "old-exe-token" + assert "secrets" not in config + assert "service_token" not in config["sandbox"] + assert "EXE_API_TOKEN" not in config["sandbox"]["exe"] def test_compose_plane_env_additions_survive_rerender(tmp_path): @@ -437,7 +490,7 @@ def test_legacy_github_table_still_validates(tmp_path): assert "GITHUB_OPERATOR_APP_ID" not in read_env(env_path) -def test_setup_toml_is_the_settings_source(tmp_path, monkeypatch): +def test_setup_toml_and_env_are_the_settings_source(tmp_path, monkeypatch): env_path = tmp_path / ".env" rc = _run( @@ -457,13 +510,16 @@ def test_setup_toml_is_the_settings_source(tmp_path, monkeypatch): assert rc == 0 toml_path = tmp_path / "druks.toml" config = _read_toml(toml_path) + values = read_env(env_path) monkeypatch.setenv("DRUKS_CONFIG", str(toml_path)) + for name in ("DRUKS_SECRETS_KEY", "DRUKS_SANDBOX_SERVICE_TOKEN"): + monkeypatch.setenv(name, values[name]) settings = Settings() assert settings.identity.model_dump() == config["identity"] assert settings.urls.model_dump() == config["urls"] - assert settings.secrets.secrets_key == config["secrets"]["secrets_key"] - assert settings.sandbox.service_token == config["sandbox"]["service_token"] + assert settings.secrets_key.get_secret_value() == values["DRUKS_SECRETS_KEY"] + assert settings.sandbox.service_token.get_secret_value() == values["SERVICE_TOKENS"] assert settings.sandbox.service_url == config["sandbox"]["service_url"] assert settings.sandbox.image == config["sandbox"]["image"] assert settings.sandbox.issuer_url == config["sandbox"]["issuer_url"] @@ -574,7 +630,7 @@ def test_a_copy_of_a_secrets_setting_is_a_named_gap(tmp_path, assignment, gap): assert rc == GAPS_EXIT_CODE assert gap in "\n".join(printed) values = read_env(env_path) - assert values["SECRETS_KEY"] == drukbox_key(tmp_path) + assert values["SECRETS_KEY"] != "wrong" assert "SECRETS_PROXY_URL" not in values diff --git a/docs/configuration.md b/docs/configuration.md index c5728130a..d6f230a19 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -10,13 +10,16 @@ without replacing the process. | Plane | Examples | Stored in | | --- | --- | --- | -| Deployment | installation timezone, identity, ingress, Drukbox, encryption key | `~/druks/druks.toml` | +| Deployment | installation timezone, identity, ingress, Drukbox | `~/druks/druks.toml` | | Dashboard | personal timezone, the GitHub connection, harness and tracker credentials, workflow and agent overrides, MCP servers, skills | Postgres | The installer creates the deployment `.env` from `druks.toml`. Compose, Druks, -and Drukbox consume this build artifact. Do not edit `.env`. Edit `druks.toml`, -then run the installer again to apply changes. `druks setup` creates `.env` but -does not restart services. +and Drukbox consume this build artifact. Edit `druks.toml`, then run the +installer again to apply changes. `druks setup` creates `.env` but does not +restart services. + +Do not edit `.env`, with one exception. Its last section holds the +[secrets](#secrets), and the installer keeps that section. The file location determines its format. Repository files such as `.druks/software_factory/config.yml` use YAML. Other repository dotfiles use @@ -38,9 +41,8 @@ host-run development template for that environment plane. | --- | --- | | `[identity]` | Browser identity mode and header or JWT verification inputs | | `[urls]` | Dashboard callback base URL and public webhook hostname | -| `[secrets]` | Generated deployment secrets | | `[paths]` | Host data and harness configuration paths | -| `[sandbox]` | Drukbox provider, service URL and token, image override, registry access, and the proxy and issuer addresses | +| `[sandbox]` | Drukbox provider, service URL, image override, registry host and user name, and the proxy and issuer addresses | | `[sandbox.]` | Provider environment passed through to the remote stack | | `[env]` | Additional deployment environment settings rendered verbatim | @@ -58,9 +60,58 @@ Every other provider name selects the generic remote shape. Drukbox validates it The local `docker` shape does not render `[sandbox.]`. Its Drukbox service gets its environment from the defaults in `deploy/compose.yaml`. -The installer generates secrets only when it first creates the TOML. When you move or -recover an installation, preserve `[secrets]`. Use repeatable -`druks setup ... --set key.path=value` arguments for explicit scripted writes. +Use repeatable `druks setup ... --set key.path=value` arguments for explicit +scripted writes. + +## Secrets + +`druks.toml` holds no secret. The secrets of Druks are `secrets_key`, +`database_url`, `redis_url`, `sandbox.service_token`, and +`sandbox.browser_login_proxy`. +Druks takes each one from an environment variable. The name is `DRUKS_` and the +key path, with `_` for each dot: `DRUKS_SECRETS_KEY`, +`DRUKS_SANDBOX_SERVICE_TOKEN`. Druks does not start when `druks.toml` holds a +secret. + +### Secrets of an installation + +The installer keeps the secrets in the last section of `~/druks/.env`: + +| Variable | Reader | From | +| --- | --- | --- | +| `DRUKS_SECRETS_KEY` | Druks | The installer. The [vault key](#credential-custody-and-secrets-at-rest) | +| `DRUKS_SANDBOX_SERVICE_TOKEN`, `SERVICE_TOKENS` | Druks, Drukbox | The installer. One token, under the name of each reader | +| `DRUKS_POSTGRES_PASSWORD` | Postgres, and the database URLs that Compose builds | The installer | +| `SECRETS_KEY` | Drukbox | The installer. The Drukbox key | +| `REGISTRY_PASSWORD` | Drukbox | You, for private sandbox images | +| `DRUKS_SANDBOX_BROWSER_LOGIN_PROXY` | Druks | You, for a login-window proxy | +| `DRUKS_DATABASE_URL`, `DRUKS_REDIS_URL` | Druks | You, only for a database or a Redis that Compose does not run | +| A provider secret, for example `EXE_API_TOKEN` | Drukbox | You | + +The installer makes its secrets one time and does not change a value that is +there. It keeps each line of the section when it renders `.env` again. To set a +secret that only you know, add its line to the section. The local shape needs no +secret from you. + +Druks does not enumerate providers. Put a plain provider variable in +`[sandbox.]` and a provider secret in the secrets section. The +installer reports a variable that is in both places. It moves a known secret +that it finds in `druks.toml` to the section, so `druks setup ... --set` also +sets one. + +Apply a changed secret with `docker compose up -d`. The exception is +`DRUKS_POSTGRES_PASSWORD`: Postgres uses it only when it creates the database. + +`.env` is the only copy of these secrets. When you move or recover an +installation, preserve it. + +### Secret files + +Druks can also read a secret from a file, for a platform that delivers secrets +as files. Set `DRUKS_SECRETS_DIR` to the directory, for example a mounted +`/run/secrets`. The file name is the key path: `secrets_key`, `redis_url`, +`sandbox.service_token`. Druks does not start when a secret is in the +environment and in a file. The installer does not use files. ## Personal and installation settings @@ -130,13 +181,13 @@ rejects execution settings. The installation API rejects timezone changes. | Variable | Default | Purpose | | --- | --- | --- | -| `DRUKS_DATABASE_URL` | local `druks` Postgres | Runtime and DBOS database | +| `DRUKS_DATABASE_URL` | local `druks` Postgres | Runtime and DBOS database. A [secret](#secrets) | | `DRUKS_DATABASE_POOL_SIZE` | `20` | Connections each process keeps open for requests and workflow steps | | `DRUKS_DATABASE_MAX_OVERFLOW` | `30` | Extra connections each process opens under load and closes after use | | `DRUKS_DBOS_POOL_SIZE` | `20` | Connections each process keeps for DBOS | | `DRUKS_TEST_DATABASE_URL` | local `druks_test` Postgres | What the shipped pytest fixtures use — never the runtime's | | `DRUKS_TEST_REDIS_URL` | `redis://127.0.0.1:6379/15` | What the shipped pytest fixtures flush | -| `DRUKS_REDIS_URL` | `redis://127.0.0.1:6379/0` | Short-lived coordination and caches | +| `DRUKS_REDIS_URL` | `redis://127.0.0.1:6379/0` | Short-lived coordination and caches. A [secret](#secrets) | | `DRUKS_DATA_DIR` | `/var/lib/druks` | Logs, artifacts, installed skills | | `DRUKS_HARNESS_CONFIG_ROOT` | `~/.config/druks/harnesses` | Optional harness configuration copied into sandboxes | | `DRUKS_LOG_LEVEL` | `INFO` | Python and DBOS log level | @@ -658,23 +709,23 @@ before provisioning a VM if its selected credential is missing. ## Sandboxes -| TOML key | Purpose | +| Key | Purpose | | --- | --- | | `sandbox.service_url` | Drukbox API base URL. An empty value disables sandbox-backed execution | -| `sandbox.service_token` | Drukbox API token | +| `sandbox.service_token` | Drukbox API token. A [secret](#secrets) | | `sandbox.timeout` | Control-plane request timeout. The default is 180 seconds | | `sandbox.image` | Optional provider image override | -| `sandbox.registry_host`, `sandbox.registry_username`, `sandbox.registry_password` | Access to private sandbox images on one registry host, for example `ghcr.io`. Set the three together. See [Drukbox](https://github.com/czpython/drukbox/blob/main/docs/deploy.md#private-image-registry) | +| `sandbox.registry_host`, `sandbox.registry_username`, `sandbox.registry_password` | Access to private sandbox images on one registry host, for example `ghcr.io`. Set the three together. The password is the secret [`REGISTRY_PASSWORD`](#secrets-of-an-installation). See [Drukbox](https://github.com/czpython/drukbox/blob/main/docs/deploy.md#private-image-registry) | | `sandbox.template_repository` | The repository path on that host where Drukbox publishes sandbox templates. The exe provider requires it | | `sandbox.proxy_url` | The secrets proxy, at the address a sandbox dials. The docker shape sets `http://172.17.0.1:8880`. docker-sbx leaves it empty | | `sandbox.issuer_url` | The issuer base URL the secrets exchange dials. The default is `http://127.0.0.1:8001`. For a Drukbox on another server, set the address of the Druks host that Drukbox reaches. The installer then serves the issuer route there ([the issuer listener](deployment.md#the-issuer-listener)) | -| `sandbox.browser_login_proxy` | Login-window egress proxy. An empty value keeps the box IP | +| `sandbox.browser_login_proxy` | Login-window egress proxy. A [secret](#secrets). With no value, the login uses the box IP | | `sandbox.browser_login_tz` | Login-window timezone (IANA zone). An empty value keeps the container default | `DRUKS_SANDBOX_KEYS_DIR` remains a process environment override for the per-host SSH private-key directory. -`[sandbox].browser_login_proxy` sends the browser **login window** through an +The secret `sandbox.browser_login_proxy` sends the browser **login window** through an HTTP proxy. The login then leaves from a different IP than the box. Use it for sign-in flows that refuse a login from the box IP. Only the login window uses the proxy. Borrowed sessions keep the box IP. This is sufficient after Druks makes @@ -698,14 +749,16 @@ are two common types. 4. Set `TS_USERSPACE=true`. 5. Set `TS_OUTBOUND_HTTP_PROXY_LISTEN=:8080`. 6. Set `TS_EXTRA_ARGS=--exit-node=`. -7. Set `browser_login_proxy = http://172.17.0.1:8080`. +7. Add `DRUKS_SANDBOX_BROWSER_LOGIN_PROXY=http://172.17.0.1:8080` to the secrets + section of `.env`. The login then leaves from your home connection. The box keeps its own IP for all other traffic. This exit needs no user name or password. **A rented static-residential (ISP) proxy.** First make sure that a detection -service does not already know the IP as a proxy. Then set the proxy with its user -name and password: `browser_login_proxy = http://user:pass@isp-host:port`. +service does not already know the IP as a proxy. Then set the secret to the proxy +with its user name and password: +`DRUKS_SANDBOX_BROWSER_LOGIN_PROXY=http://user:pass@isp-host:port`. An ISP IP passes the datacenter-ASN check. A detection service can still find it and mark it as a proxy. @@ -858,7 +911,7 @@ kind, an audience, and an encrypted mapping of secrets: A revoked row keeps its facts and loses its secrets. An agent call keeps its reference to the row it billed. A reconnect revives the row. -`secrets.secrets_key` encrypts the vault and the browser-session payloads with +The secret `secrets_key` encrypts the vault and the browser-session payloads with AES-256-GCM. Each database column supplies authenticated associated data, and each value gets a derived encryption key. The setting is one or more comma-separated, base64-encoded 32-byte master keys: @@ -868,11 +921,10 @@ python3 -c 'import base64, os; print(base64.b64encode(os.urandom(32)).decode())' ``` The first key encrypts new values. Each listed key can decrypt values. To rotate -the key, put a new key first in `druks.toml`. Then run the installer again: +the key, put a new key first in `.env`. Then run `docker compose up -d`: -```toml -[secrets] -secrets_key = "," +```bash +DRUKS_SECRETS_KEY=, ``` While a stored row depends on the old key, keep that key. If you lose each key @@ -881,10 +933,9 @@ subscriptions. Enter the static tokens again. Log in to the affected browser sessions again. Validation and API errors do not include submitted secret values. -`secrets.drukbox_secrets_key` encrypts the secret entries of each sandbox in -the Drukbox database. The installer generates it and renders it as -`SECRETS_KEY` for the Drukbox API and the secrets exchange. Rotate it as you -rotate `secrets_key`, with the new key first. +`SECRETS_KEY` in `.env` encrypts the secret entries of each sandbox in the +Drukbox database. The installer generates it for the Drukbox API and the secrets +exchange. Rotate it as you rotate `secrets_key`, with the new key first. The envelope does **not** cover notification webhook URLs. Postgres stores them as ordinary fields, although the API masks their values. Treat access to diff --git a/docs/deployment.md b/docs/deployment.md index 66bd4ac1e..bced9f283 100644 --- a/docs/deployment.md +++ b/docs/deployment.md @@ -71,12 +71,13 @@ curl -fsSL https://druks.ai/install.sh | DRUKS_PROVIDER=exe bash `docker` is the default provider for the local shape. A remote deployment names its provider. Use `exe` for exe.dev. Use another Drukbox provider name for the -generic remote shape. The first pass writes `~/druks/druks.toml` with generated -secrets. +generic remote shape. The first pass writes `~/druks/druks.toml`. -It creates `~/druks/.env` and exits if required values are missing. -The output identifies each missing value. Edit `druks.toml`. For a generic -remote shape, fill `[sandbox.]` from the Drukbox +It creates `~/druks/.env` with generated secrets, and exits if required values +are missing. The output identifies each missing value. Edit `druks.toml`, and +add each missing secret to the +[secrets section](configuration.md#secrets-of-an-installation) of `.env`. For a +generic remote shape, fill `[sandbox.]` from the Drukbox [configuration reference](https://github.com/czpython/drukbox). The installer also creates `paths.harness_config_root`. Put optional CLI configuration in the harness directory under that root. @@ -276,14 +277,14 @@ Drukbox doctor checks the exchange, and `druks doctor` shows that result in its `drukbox` check. Drukbox encrypts the secret entries of each sandbox with `SECRETS_KEY`. The -installer generates `[secrets].drukbox_secrets_key` and renders it as -`SECRETS_KEY` for the API and the exchange. Pin the proxy image with +installer generates `SECRETS_KEY` in `.env` for the API and the exchange. Pin +the proxy image with `DRUKS_SECRETS_PROXY_IMAGE` in `[env]`, at the tag of `DRUKS_SANDBOX_SERVICE_IMAGE`. An install from before these services has `SECRETS_KEY` in `[env]` or in -`[sandbox.]`. Move that value to `[secrets].drukbox_secrets_key`, -set `[sandbox].proxy_url`, and run the installer again. +`[sandbox.]`. Move that value to the `SECRETS_KEY` line in the secrets +section of `.env`, set `[sandbox].proxy_url`, and run the installer again. ### The issuer listener diff --git a/docs/development.md b/docs/development.md index df2f309c5..0ba574cc6 100644 --- a/docs/development.md +++ b/docs/development.md @@ -20,7 +20,7 @@ cp .env.example .env python3 -c 'import base64, os; print(base64.b64encode(os.urandom(32)).decode())' ``` -Paste the generated value into `secrets.secrets_key` in `druks.toml`, then +Paste the generated value into `DRUKS_SECRETS_KEY` in `.env`, then initialize the development database: ```bash @@ -181,10 +181,11 @@ Drukbox on the host from its own checkout ```toml [sandbox] service_url = "http://127.0.0.1:8000" -service_token = "dev-token" image = "ghcr.io/czpython/druks/sandbox:latest" ``` +The token is a secret. Set `DRUKS_SANDBOX_SERVICE_TOKEN=dev-token` in `.env`. + `uv run druks doctor --sandbox` creates a real host. If you require a real sandbox test, run this command. It is not part of the normal test suite. diff --git a/docs/full-local.md b/docs/full-local.md index d7e52df10..70d6e0c6c 100644 --- a/docs/full-local.md +++ b/docs/full-local.md @@ -111,7 +111,7 @@ authentication and exactly one operator account. A new installation shows its setup page until the first subscription connection completes. That connection creates the operator account from the provider-verified email. Protect database -access and backups as credential data. The `[secrets].secrets_key` envelope +access and backups as credential data. The `secrets_key` envelope protects every secret in the vault, subscriptions included. Agent calls refuse before provisioning if their selected harness is not diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md index 122b876f6..21b4662a5 100644 --- a/docs/troubleshooting.md +++ b/docs/troubleshooting.md @@ -34,7 +34,8 @@ provider capacity and can take about a VM minute. Read the named field in the error. Common causes: -- **Secrets key:** `secrets.secrets_key` is empty, has invalid base64, or does not decode to 32 bytes. +- **Secrets key:** `DRUKS_SECRETS_KEY` is empty, has invalid base64, or does not decode to 32 bytes. +- **A secret in `druks.toml`:** the error names the key. Run the installer again, and it moves the value to `.env`. - A `druks.toml` value creates an invalid process setting. Re-running `install.sh` renders `.env` from `druks.toml` and prints remaining diff --git a/druks.toml.example b/druks.toml.example index 20633cbcd..a55b5efda 100644 --- a/druks.toml.example +++ b/druks.toml.example @@ -1,5 +1,6 @@ # Host-run development configuration. Copy to `druks.toml`. -# Reference: docs/configuration.md. +# Reference: docs/configuration.md. This file holds no secret: a host-run server +# takes each secret from `.env` (docs/development.md). # Schedule timezone and initial timezone for new accounts. timezone = "UTC" @@ -16,13 +17,8 @@ jwt_identity_claim = "/email" endpoint = "" webhook_host = "" -[secrets] -# python3 -c 'import base64, os; print(base64.b64encode(os.urandom(32)).decode())' -secrets_key = "" - [sandbox] service_url = "" -service_token = "" image = "" issuer_url = "" timeout = 180 diff --git a/scripts/install.sh b/scripts/install.sh index 5def84e54..09792b201 100755 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -4,7 +4,7 @@ # Non-interactive and idempotent: re-run any time to pull a fresh # compose.yaml + new images; re-running is also the upgrade path. # Deployment configuration lives in druks.toml; ``druks setup`` (run from the -# backend image) creates it with generated secrets and renders .env. When +# backend image) creates it and renders .env with generated secrets. When # everything needed to boot is present the same run migrates the DB (out of # band, once — never on boot) and brings the stack up; otherwise it prints # the remaining checklist and exits. GitHub and the coding CLIs connect from