From 0c2049cc9c88a5a7399e218c11e45f86363eb038 Mon Sep 17 00:00:00 2001 From: Daniil Anfimov Date: Sat, 27 Jun 2026 17:39:04 +0300 Subject: [PATCH 1/3] Stop templating alembic.ini; use repo file with env-sourced DB URL albs-web-server now sets alembic's sqlalchemy.url at runtime in alws/alembic/env.py from settings.sync_database_url (SYNC_DATABASE_URL), and dropped the hardcoded URL from alws/alembic.ini. That URL was the only per-deployment value the deploy templated, and SYNC_DATABASE_URL is already provided via vars.env, so alembic.ini.j2 is now redundant. Use the upstream alws/alembic.ini directly, matching the existing nginx config approach. --- roles/dev_deploy/defaults/main/configs.yml | 9 +- roles/dev_deploy/templates/alembic.ini.j2 | 100 --------------------- 2 files changed, 5 insertions(+), 104 deletions(-) delete mode 100644 roles/dev_deploy/templates/alembic.ini.j2 diff --git a/roles/dev_deploy/defaults/main/configs.yml b/roles/dev_deploy/defaults/main/configs.yml index 193642a..2f21948 100644 --- a/roles/dev_deploy/defaults/main/configs.yml +++ b/roles/dev_deploy/defaults/main/configs.yml @@ -8,10 +8,11 @@ generated_configs: # in the albs-web-server repo at nginx_configs/albs.conf and is mounted # directly into the nginx container. It has no per-deployment values, so # there is no reason to duplicate it as a Jinja template here. - - sname: alembic.ini.j2 - dname: alembic.ini - dest: "{{ sources_root }}/albs-web-server/alws" - mode: "0644" + # alembic.ini is intentionally NOT templated here — it lives in the + # albs-web-server repo at alws/alembic.ini and is used directly. The DB + # URL is no longer hardcoded in it; alws/alembic/env.py sets it at runtime + # from settings.sync_database_url (SYNC_DATABASE_URL), which is provided + # via vars.env. There are no per-deployment values left to template. - sname: settings.py.j2 dname: settings.py dest: "{{ volumes_root }}/pulp/settings" diff --git a/roles/dev_deploy/templates/alembic.ini.j2 b/roles/dev_deploy/templates/alembic.ini.j2 deleted file mode 100644 index 524051d..0000000 --- a/roles/dev_deploy/templates/alembic.ini.j2 +++ /dev/null @@ -1,100 +0,0 @@ -# A generic, single database configuration. - -[alembic] -# path to migration scripts -script_location = alws/alembic - -# template used to generate migration files -# file_template = %%(rev)s_%%(slug)s - -# sys.path path, will be prepended to sys.path if present. -# defaults to the current working directory. -prepend_sys_path = . - -# timezone to use when rendering the date within the migration file -# as well as the filename. -# If specified, requires the python-dateutil library that can be -# installed by adding `alembic[tz]` to the pip requirements -# string value is passed to dateutil.tz.gettz() -# leave blank for localtime -# timezone = - -# max length of characters to apply to the -# "slug" field -# truncate_slug_length = 40 - -# set to 'true' to run the environment during -# the 'revision' command, regardless of autogenerate -# revision_environment = false - -# set to 'true' to allow .pyc and .pyo files without -# a source .py file to be detected as revisions in the -# versions/ directory -# sourceless = false - -# version location specification; This defaults -# to alembic/versions. When using multiple version -# directories, initial revisions must be specified with --version-path. -# The path separator used here should be the separator specified by "version_path_separator" -# version_locations = %(here)s/bar:%(here)s/bat:alembic/versions - -# version path separator; As mentioned above, this is the character used to split -# version_locations. Valid values are: -# -# version_path_separator = : -# version_path_separator = ; -# version_path_separator = space -version_path_separator = os # default: use os.pathsep - -# the output encoding used when revision files -# are written from script.py.mako -# output_encoding = utf-8 - -sqlalchemy.url = {{ albs_db_sync_url }} - - -[post_write_hooks] -# post_write_hooks defines scripts or Python functions that are run -# on newly generated revision scripts. See the documentation for further -# detail and examples - -# format using "black" - use the console_scripts runner, against the "black" entrypoint -# hooks = black -# black.type = console_scripts -# black.entrypoint = black -# black.options = -l 79 REVISION_SCRIPT_FILENAME - -# Logging configuration -[loggers] -keys = root,sqlalchemy,alembic - -[handlers] -keys = console - -[formatters] -keys = generic - -[logger_root] -level = WARN -handlers = console -qualname = - -[logger_sqlalchemy] -level = WARN -handlers = -qualname = sqlalchemy.engine - -[logger_alembic] -level = INFO -handlers = -qualname = alembic - -[handler_console] -class = StreamHandler -args = (sys.stderr,) -level = NOTSET -formatter = generic - -[formatter_generic] -format = %(levelname)-5.5s [%(name)s] %(message)s -datefmt = %H:%M:%S \ No newline at end of file From 9b6bd2751cf4cd2d9325863a536908f7bf2a1338 Mon Sep 17 00:00:00 2001 From: Daniil Anfimov Date: Sat, 27 Jun 2026 17:52:27 +0300 Subject: [PATCH 2/3] Adapt dev deploy to Pulp multi-container stack albs-web-server now runs Pulp as a multi-container stack (pulp-minimal + pulp-web) instead of the all-in-one pulp/pulp image. Adjust the deploy to match: - Point the Pulp DB URLs at the separate postgres service (postgres:5432) instead of the old all-in-one pulp host, which is now HTTP-only. - Stop templating Pulp's settings.py: it now ships in the web-server repo at assets/pulp/settings.py and is mounted directly; volumes_root/pulp/ settings is no longer mounted. Drop the template and its directory. - Retarget the admin password reset to the pulp_api container and drop the obsolete 'enable pulp database external access' and pulp stop/start tasks: the postgres:16 service already allows network access (scram-sha-256 via pulp-postgres.env) and stores data under /var/lib/postgresql/data. --- inventories/one_vm/group_vars/all.yml | 8 +++-- roles/dev_deploy/defaults/main/common.yml | 2 -- roles/dev_deploy/defaults/main/configs.yml | 8 ++--- roles/dev_deploy/tasks/misc.yml | 37 ++++------------------ roles/dev_deploy/templates/settings.py.j2 | 6 ---- 5 files changed, 17 insertions(+), 44 deletions(-) delete mode 100644 roles/dev_deploy/templates/settings.py.j2 diff --git a/inventories/one_vm/group_vars/all.yml b/inventories/one_vm/group_vars/all.yml index 43fcc26..0976720 100644 --- a/inventories/one_vm/group_vars/all.yml +++ b/inventories/one_vm/group_vars/all.yml @@ -39,8 +39,12 @@ rabbitmq_vhost: "rabbitmq" pulp_host: "http://pulp" pulp_password: "password" pulp_user: "admin" -pulp_database_url: "postgresql+psycopg2://postgres:{{ pulp_password }}@pulp/pulp" -async_pulp_database_url: "postgresql+asyncpg://postgres:{{ pulp_password }}@pulp/pulp" +# Pulp runs as a multi-container stack: the "pulp" host (pulp_web alias) is +# HTTP-only, and the Pulp DB lives in the separate "postgres" service. The DB +# user/password here must match pulp-postgres.env in the albs-web-server repo +# (postgres/password). +pulp_database_url: "postgresql+psycopg2://postgres:{{ pulp_password }}@postgres:5432/pulp" +async_pulp_database_url: "postgresql+asyncpg://postgres:{{ pulp_password }}@postgres:5432/pulp" immudb_username: "" immudb_password: "" diff --git a/roles/dev_deploy/defaults/main/common.yml b/roles/dev_deploy/defaults/main/common.yml index 84a4f61..239b18c 100644 --- a/roles/dev_deploy/defaults/main/common.yml +++ b/roles/dev_deploy/defaults/main/common.yml @@ -12,8 +12,6 @@ created_directories: mode: "0755" # - path: "{{ volumes_root }}/{{ folder_name_for_result_backend }}" # mode: "0755" - - path: "{{ volumes_root }}/pulp/settings" - mode: "0755" - path: "{{ sources_root }}/albs-sign-node/node-config" mode: "0755" - path: "{{ sources_root }}/albs-node/node-config/castor" diff --git a/roles/dev_deploy/defaults/main/configs.yml b/roles/dev_deploy/defaults/main/configs.yml index 2f21948..b72ddd1 100644 --- a/roles/dev_deploy/defaults/main/configs.yml +++ b/roles/dev_deploy/defaults/main/configs.yml @@ -13,10 +13,10 @@ generated_configs: # URL is no longer hardcoded in it; alws/alembic/env.py sets it at runtime # from settings.sync_database_url (SYNC_DATABASE_URL), which is provided # via vars.env. There are no per-deployment values left to template. - - sname: settings.py.j2 - dname: settings.py - dest: "{{ volumes_root }}/pulp/settings" - mode: "0644" + # Pulp's settings.py is intentionally NOT templated here — it lives in the + # albs-web-server repo at assets/pulp/settings.py and is mounted directly into + # every Pulp service by docker-compose. The multi-container stack no longer + # mounts volumes_root/pulp/settings, and the file has no per-deployment values. - sname: albs-gitea-listener-config.yaml.j2 dname: albs-gitea-listener-config.yaml dest: "{{ sources_root }}/albs-web-server/alws/scripts/albs-gitea-listener" diff --git a/roles/dev_deploy/tasks/misc.yml b/roles/dev_deploy/tasks/misc.yml index 49a1ba7..d3bdb7f 100644 --- a/roles/dev_deploy/tasks/misc.yml +++ b/roles/dev_deploy/tasks/misc.yml @@ -5,9 +5,15 @@ path: "{{ volumes_root }}/is_clean_installation" register: is_clean_installation +# Pulp runs as a multi-container stack: pulpcore-manager lives in the pulp_api +# service. The init admin password is seeded to "password" by the +# pulp_set_init_password one-shot in docker-compose; this resets it to the +# configured pulp_password if a custom one was provided. The separate postgres +# service already allows external access over the compose network (scram-sha-256 +# via pulp-postgres.env), so no manual pg_hba/postgresql.conf edits are needed. - name: Change pulp password community.docker.docker_container_exec: - container: "{{ container_name_prefix }}_pulp_1" + container: "{{ container_name_prefix }}_pulp_api_1" command: "bash -c 'pulpcore-manager reset-admin-password -p {{ pulp_password }}'" register: result until: result is succeeded @@ -15,35 +21,6 @@ delay: 30 when: is_clean_installation.stat.exists == False -- name: Enable pulp database external access - community.docker.docker_container_exec: - container: "{{ container_name_prefix }}_pulp_1" - command: >- - bash -c " - if ! grep -q 'listen_addresses = \*' /var/lib/pgsql/data/postgresql.conf; then - echo \"listen_addresses = '*'\" >> /var/lib/pgsql/data/postgresql.conf; - fi && - if ! grep -q 'host all all 0.0.0.0/0 md5' /var/lib/pgsql/data/pg_hba.conf; then - echo \"host all all 0.0.0.0/0 md5\" >> /var/lib/pgsql/data/pg_hba.conf; - fi && - if ! grep -q 'host all all ::/0 md5' /var/lib/pgsql/data/pg_hba.conf; then - echo \"host all all ::/0 md5\" >> /var/lib/pgsql/data/pg_hba.conf; - fi && - runuser postgres -c 'echo \"ALTER USER postgres WITH PASSWORD '\''{{ pulp_password }}'\'';\" | /usr/bin/psql'" - when: is_clean_installation.stat.exists == False - -- name: Stop pulp container - community.docker.docker_container: - name: "{{ container_name_prefix }}_pulp_1" - state: stopped - when: is_clean_installation.stat.exists == False - -- name: Start pulp container - community.docker.docker_container: - name: "{{ container_name_prefix }}_pulp_1" - state: started - when: is_clean_installation.stat.exists == False - - name: Wait for web_server container to be running community.docker.docker_container_info: name: "{{ container_name_prefix }}_web_server_1" diff --git a/roles/dev_deploy/templates/settings.py.j2 b/roles/dev_deploy/templates/settings.py.j2 deleted file mode 100644 index 0ca50c3..0000000 --- a/roles/dev_deploy/templates/settings.py.j2 +++ /dev/null @@ -1,6 +0,0 @@ -CONTENT_ORIGIN='http://pulp' -ANSIBLE_API_HOSTNAME='http://pulp' -ANSIBLE_CONTENT_HOSTNAME='http://pulp/pulp/content' -TOKEN_AUTH_DISABLED=True -ALLOW_AUTOMATIC_UNSAFE_ADVISORY_CONFLICT_RESOLUTION=True -ALLOWED_EXPORT_PATHS=['/srv/exports'] From d653170b2e5522f1f767cec0e027dc22a30a44a8 Mon Sep 17 00:00:00 2001 From: Daniil Anfimov Date: Sat, 27 Jun 2026 18:21:00 +0300 Subject: [PATCH 3/3] Build/start compose services in parallel instead of per-service loop The per-service Ansible loop invoked 'docker compose up --build' once per service, serializing image builds. Replace it with a single compose invocation that takes all non-excluded services as arguments, so BuildKit builds in parallel and startup still respects depends_on. - services.yml: filter the service list (minus excluded_containers and pre_bootstrap_excluded_containers) and pass it in one 'up -d --build'. - misc.yml: same for the post-JWT consumer-container recreate and the deferred pre-bootstrap container start. The filtered list is guarded with a length check so an empty argument list can never cause 'docker compose up' to act on ALL services. --compatibility is retained so container names keep the underscore (v1) form the name-based tasks in misc.yml depend on. --- roles/dev_deploy/tasks/misc.yml | 23 +++++++++++++++++------ roles/dev_deploy/tasks/services.yml | 22 ++++++++++++++++------ 2 files changed, 33 insertions(+), 12 deletions(-) diff --git a/roles/dev_deploy/tasks/misc.yml b/roles/dev_deploy/tasks/misc.yml index d3bdb7f..c083f3d 100644 --- a/roles/dev_deploy/tasks/misc.yml +++ b/roles/dev_deploy/tasks/misc.yml @@ -97,27 +97,38 @@ - regenerated_albs_jwt is defined - regenerated_albs_jwt.stdout | default('') | length > 0 +# Recreate all non-excluded consumer containers in a single compose invocation +# (parallel, depends_on-ordered) instead of a per-service loop. The filtered list +# is guarded so an empty arg list never makes 'docker compose up' touch ALL +# services (including the excluded ones). - name: Recreate consumer containers to pick up regenerated JWT tags: - jwt_tokens - albs_jwt_token - services - shell: "{{ docker_compose }} -p {{ container_name_prefix }} --compatibility up -d --force-recreate {{ item }}" + vars: + consumer_services: >- + {{ services.stdout_lines + | difference(excluded_containers) + | difference(pre_bootstrap_excluded_containers | default([])) }} + shell: >- + {{ docker_compose }} -p {{ container_name_prefix }} --compatibility + up -d --force-recreate {{ consumer_services | join(' ') }} args: chdir: "{{ sources_root }}/albs-web-server" - loop: "{{ services.stdout_lines }}" when: - regenerated_albs_jwt is defined - regenerated_albs_jwt.stdout | default('') | length > 0 - - item not in excluded_containers - - item not in (pre_bootstrap_excluded_containers | default([])) - services is defined + - consumer_services | length > 0 - name: Start containers deferred until after bootstrap - shell: "{{ docker_compose }} -p {{ container_name_prefix }} --compatibility up -d --build --force-recreate {{ item }}" + shell: >- + {{ docker_compose }} -p {{ container_name_prefix }} --compatibility + up -d --build --force-recreate + {{ pre_bootstrap_excluded_containers | default([]) | join(' ') }} args: chdir: "{{ sources_root }}/albs-web-server" - loop: "{{ pre_bootstrap_excluded_containers | default([]) }}" when: pre_bootstrap_excluded_containers | default([]) | length > 0 - name: Checking if GPG key exists on web_server diff --git a/roles/dev_deploy/tasks/services.yml b/roles/dev_deploy/tasks/services.yml index 6656a47..87b71f9 100644 --- a/roles/dev_deploy/tasks/services.yml +++ b/roles/dev_deploy/tasks/services.yml @@ -16,14 +16,24 @@ chdir: "{{ sources_root }}/albs-web-server" register: services - - name: Create and start each service individually - shell: "{{ docker_compose }} -p {{ container_name_prefix }} --compatibility up -d --build --force-recreate {{ item }}" + # Start all non-excluded services in a single compose invocation so images + # build in parallel (BuildKit) and startup respects depends_on. A per-service + # loop would serialize the builds. Excluded containers are filtered out of the + # service list rather than skipped via 'docker compose up' (which has no + # native "all except X" flag). The filtered list is guarded so an empty arg + # list never makes 'docker compose up' start ALL services. + - name: Create and start non-excluded services + vars: + included_services: >- + {{ services.stdout_lines + | difference(excluded_containers) + | difference(pre_bootstrap_excluded_containers | default([])) }} + shell: >- + {{ docker_compose }} -p {{ container_name_prefix }} --compatibility + up -d --build --force-recreate {{ included_services | join(' ') }} args: chdir: "{{ sources_root }}/albs-web-server" - when: - - item not in excluded_containers - - item not in (pre_bootstrap_excluded_containers | default([])) - loop: "{{ services.stdout_lines }}" + when: included_services | length > 0 when: - docker_compose != "not found" - (excluded_containers | length > 0) or (pre_bootstrap_excluded_containers | default([]) | length > 0)