From 9f7580b3b5709ee48372ea274d3077f7e6848e89 Mon Sep 17 00:00:00 2001 From: mnemonik-dev Date: Thu, 9 Jul 2026 02:18:50 +0300 Subject: [PATCH 01/19] feat(mnemonik-server): co-located Mnemonic MCP server ansible role Deploys the SERVER side of the Mnemonic protocol (Rust MCP image from GHCR + local Ollama) co-located on the fabric VM, fronted by the shared Caddy instead of the monorepo's own nginx/certbot. Distinct from the client-binary mnemonic-mcp role. - role: compose (mcp + ollama, nginx/certbot dropped), state bind-mounted on the persistent volume, GHCR pull, ollama model pull, Caddy vhost, health gate - deploy.yml: gated behind mnemonik_server_enabled (default false), secrets threaded from sops (vault_mnemonik_mcp_jwt_secret / _refresh_salt) - secrets template: documents the two hosted-mode secrets - 8 contract tests, ansible-lint at sibling baseline, playbook syntax-check ok --- infrastructure/ansible/playbooks/deploy.yml | 20 ++ .../roles/mnemonik-server/.ansible-lint | 8 + .../ansible/roles/mnemonik-server/README.md | 81 ++++++ .../roles/mnemonik-server/defaults/main.yml | 90 +++++++ .../roles/mnemonik-server/handlers/main.yml | 22 ++ .../roles/mnemonik-server/meta/main.yml | 17 ++ .../roles/mnemonik-server/tasks/main.yml | 231 ++++++++++++++++++ .../templates/Caddyfile.snippet.j2 | 20 ++ .../templates/docker-compose.yml.j2 | 79 ++++++ .../mnemonik-server/templates/mcp.env.j2 | 23 ++ .../tests/test_role_contract.py | 100 ++++++++ .../secrets/secrets.sops.yml.template | 11 + 12 files changed, 702 insertions(+) create mode 100644 infrastructure/ansible/roles/mnemonik-server/.ansible-lint create mode 100644 infrastructure/ansible/roles/mnemonik-server/README.md create mode 100644 infrastructure/ansible/roles/mnemonik-server/defaults/main.yml create mode 100644 infrastructure/ansible/roles/mnemonik-server/handlers/main.yml create mode 100644 infrastructure/ansible/roles/mnemonik-server/meta/main.yml create mode 100644 infrastructure/ansible/roles/mnemonik-server/tasks/main.yml create mode 100644 infrastructure/ansible/roles/mnemonik-server/templates/Caddyfile.snippet.j2 create mode 100644 infrastructure/ansible/roles/mnemonik-server/templates/docker-compose.yml.j2 create mode 100644 infrastructure/ansible/roles/mnemonik-server/templates/mcp.env.j2 create mode 100644 infrastructure/ansible/roles/mnemonik-server/tests/test_role_contract.py diff --git a/infrastructure/ansible/playbooks/deploy.yml b/infrastructure/ansible/playbooks/deploy.yml index 8faceb3..35784f3 100644 --- a/infrastructure/ansible/playbooks/deploy.yml +++ b/infrastructure/ansible/playbooks/deploy.yml @@ -114,6 +114,11 @@ vault_blogger_repo_ref: "{{ sops.blogger_repo_ref | default('') }}" vault_claude_blog_repo_ref: "{{ sops.claude_blog_repo_ref | default('') }}" vault_publish_callback_hmac_secrets: "{{ sops.publish_callback_hmac_secrets | default('') }}" + # Mnemonic MCP server (mnemonik-server role) hosted-mode secrets. + # Generate each once (openssl rand -base64 32) and keep stable; + # empty/safe default so a deploy with the role disabled never fails. + vault_mnemonik_mcp_jwt_secret: "{{ sops.mnemonik_mcp_jwt_secret | default('') }}" + vault_mnemonik_mcp_refresh_salt: "{{ sops.mnemonik_mcp_refresh_salt | default('') }}" no_log: true - name: Wait for system to stabilize post-boot @@ -180,6 +185,21 @@ # at the VM's public IPv4. Empty = tailnet-only (default). kaneo_public_domain: "{{ sops.kaneo_public_domain | default('kaneo.mnemonik.xyz') }}" + # Mnemonic protocol SERVER (Rust MCP image from GHCR + local Ollama), + # co-located and fronted by the shared Caddy. Distinct from the + # `mnemonic-mcp` role (that installs the CLIENT binary). Disabled by + # default: enable once DNS (mcp.mnemonik.xyz → VM IP) + sops secrets are + # in place — set mnemonik_server_enabled=true (group_vars or -e). + - role: mnemonik-server + tags: [mnemonik-server, mcp] + when: mnemonik_server_enabled | default(false) | bool + vars: + mnemonik_mcp_jwt_secret: "{{ vault_mnemonik_mcp_jwt_secret | default('') }}" + mnemonik_mcp_refresh_salt: "{{ vault_mnemonik_mcp_refresh_salt | default('') }}" + # GHCR pull auth — reuse the sops github PAT for private packages. + # Leave the token empty if the GHCR package is public. + mnemonik_ghcr_token: "{{ vault_github_deploy_pat | default('') }}" + - role: telegram-init tags: [telegram-init, communication] vars: diff --git a/infrastructure/ansible/roles/mnemonik-server/.ansible-lint b/infrastructure/ansible/roles/mnemonik-server/.ansible-lint new file mode 100644 index 0000000..0258334 --- /dev/null +++ b/infrastructure/ansible/roles/mnemonik-server/.ansible-lint @@ -0,0 +1,8 @@ +--- +# Role-local ansible-lint config. Mirrors the mnemonic-mcp role. +# +# role-name: every role in this repo uses kebab-case (mnemonic-mcp, kaneo, +# telegram-init, ssh-hardening, ...). Matching that convention trips the +# lint's snake_case rule; renaming is a repo-wide concern, out of scope here. +skip_list: + - role-name diff --git a/infrastructure/ansible/roles/mnemonik-server/README.md b/infrastructure/ansible/roles/mnemonik-server/README.md new file mode 100644 index 0000000..c12ec84 --- /dev/null +++ b/infrastructure/ansible/roles/mnemonik-server/README.md @@ -0,0 +1,81 @@ +# Role: mnemonik-server + +Deploys the **server side** of the Mnemonic protocol, co-located on the fabric +VM: the Rust MCP HTTP server (GHCR image `ghcr.io/mnemonik-xyz/mnemonic-mcp`) +plus a local Ollama for `/chat`. TLS + public ingress are handled by the +**shared Caddy** (vaultwarden role), not the monorepo's own nginx/certbot. + +> **Not** the `mnemonic-mcp` role. That one installs the *client* binary +> (`@mnemonik-xyz/mcp`, per-spawn stdio) that content-publisher spawns. This +> role hosts the *server* those clients connect to. Two sides of one protocol. + +## What it does + +1. **Swap** — idempotent `{{ mnemonik_server_swap_size_mb }}`MB swapfile + (ollama + fastembed headroom). Toggle with `mnemonik_server_swap_enabled`. +2. **State on the persistent volume** — creates `/mnemonik/{data,keypair,ollama}` + and bind-mounts them into the containers so server identity (`id.json` in + `/keypair`), the attestation DB (`/data`), and the pulled model survive a + VM rebuild. +3. **GHCR pull** — optional `docker login` (private package), then + `docker compose pull` — the box never compiles Rust. +4. **Ollama** — stock `ollama/ollama` image + a post-up `ollama pull` of + `{{ mnemonik_ollama_model }}` (replicates the monorepo's custom image + without needing its build context) + a non-fatal warm-up. +5. **Caddy vhost** — drops `mnemonik-mcp.conf` into the shared conf.d and + restarts Caddy; `{{ mnemonik_public_domain }}` gets a real Let's Encrypt + cert on first request. +6. **Health gate** — polls `https://{{ mnemonik_public_domain }}/health` the + same way an external client would. + +## Boundary with the monorepo pipeline + +- **Build stays upstream.** `mnemonik-xyz/monorepo` → `build-mcp-image.yml` + builds + pushes the image to GHCR. Untouched by this role. +- **Deploy is fabric's.** This role does what `deploy-mcp.yml`'s deploy half + does (pull + compose up + health gate), inside fabric's existing deploy that + already has authenticated access to this box — instead of maintaining the + monorepo's separate SSH path into a box whose port 22 is firewalled. +- **Webapp is elsewhere.** The frontend lives on Cloudflare Pages; only the + MCP host is served here. + +## Enabling (first deploy) + +1. **DNS** — A record `mcp.mnemonik.xyz` → VM public IPv4. +2. **sops** — `sops edit infrastructure/secrets/secrets.sops.yml`, set + `mnemonik_mcp_jwt_secret` + `mnemonik_mcp_refresh_salt` + (`openssl rand -base64 32` each, keep stable). Also confirm + `caddy_acme_email` is set so Let's Encrypt can issue. +3. **GHCR** — make the package public, or rely on the sops github PAT wired in + `deploy.yml` (`mnemonik_ghcr_token`). +4. **Enable** — set `mnemonik_server_enabled: true` (group_vars or `-e`) and + deploy. + +## Key variables + +| Variable | Default | Notes | +|----------|---------|-------| +| `mnemonik_server_enabled` | `false` | Master switch. | +| `mnemonik_mcp_image_tag` | `latest` | Pin a real tag for reproducible redeploys. | +| `mnemonik_ollama_model` | `qwen2.5:3b` | Pulled once, cached on the volume. | +| `mnemonik_public_domain` | `mcp.mnemonik.xyz` | Caddy public vhost. | +| `mnemonik_server_swap_size_mb` | `4096` | Swap cushion. | + +`mnemonik_mcp_jwt_secret` / `mnemonik_mcp_refresh_salt` have no default — the +role fails loudly if enabled without them. + +## Cross-role contract + +Depends on the **vaultwarden** role for the shared Caddy container +(`{{ vaultwarden_caddy_container_name }}`) and its external docker network +(`{{ mnemonik_shared_caddy_network }}`). The mcp container joins that network +with alias `mnemonik-mcp`; Caddy reverse-proxies to it by name. Must run after +vaultwarden + kaneo in the `deploy.yml` roles list. + +## Smoke (manual, post-deploy) + +```bash +curl -fsS https://mcp.mnemonik.xyz/health +ssh 'docker compose -f /opt/mnemonik-server/docker-compose.yml ps' +ssh 'docker exec mnemonik-ollama-1 ollama list' # model present +``` diff --git a/infrastructure/ansible/roles/mnemonik-server/defaults/main.yml b/infrastructure/ansible/roles/mnemonik-server/defaults/main.yml new file mode 100644 index 0000000..8382ab0 --- /dev/null +++ b/infrastructure/ansible/roles/mnemonik-server/defaults/main.yml @@ -0,0 +1,90 @@ +--- +# Role: mnemonik-server +# +# Deploys the SERVER side of the Mnemonic protocol — the Rust MCP HTTP server +# (GHCR image ghcr.io/mnemonik-xyz/mnemonic-mcp) plus a co-located Ollama for +# /chat — co-located on the fabric VM and fronted by the existing shared Caddy +# (vaultwarden role), NOT by the monorepo's own nginx+certbot. +# +# NOTE: distinct from the `mnemonic-mcp` role. That role installs the CLIENT +# binary (@mnemonik-xyz/mcp, per-spawn stdio) consumed by content-publisher. +# This role hosts the SERVER the protocol clients connect to. +# +# Upstream reference (adapted): mnemonik-xyz/monorepo docker-compose.yml + +# mcp/deploy/compose.env.example. We keep the `mcp` + `ollama` services and +# drop `nginx` + `certbot` (Caddy terminates TLS and reverse-proxies instead). + +# Master switch. Disabled by default: enable per-environment (group_vars or +# CLI -e) once DNS + sops secrets are in place. Mirrors the mnemonic-mcp / +# restic gating pattern. +mnemonik_server_enabled: false + +# --- Deploy dir + compose project ------------------------------------------ +mnemonik_server_install_dir: /opt/mnemonik-server +mnemonik_server_compose_project: mnemonik + +# --- MCP server image ------------------------------------------------------ +# CI (monorepo build-mcp-image.yml) builds + pushes this to GHCR; the fabric +# box only PULLS — it never compiles Rust. Pin a real tag (e.g. v0.2.8) for +# reproducible redeploys instead of `latest`. +mnemonik_mcp_image: "ghcr.io/mnemonik-xyz/mnemonic-mcp" +mnemonik_mcp_image_tag: "latest" + +# GHCR pull auth. If the package is private, the role does `docker login` +# using this token (reuse the sops github PAT). Leave empty if the GHCR +# package is public (no login attempted). +mnemonik_ghcr_username: "mnemonik-dev" +mnemonik_ghcr_token: "" + +# --- Ollama (co-located, local inference) ---------------------------------- +# Stock upstream image + a post-up `ollama pull` (the monorepo's custom +# `monorepo-ollama` image is just this base + a pre-pulled model + warm-up; +# we replicate that without needing the monorepo build context on the box). +mnemonik_ollama_image: "ollama/ollama:0.21.2" +mnemonik_ollama_model: "qwen2.5:3b" + +# --- MCP runtime env (passed into the mcp container) ----------------------- +mnemonik_mcp_storage_mode: "local" +mnemonik_mcp_embed_provider: "fastembed" +mnemonik_mcp_turbo_bits: 4 +mnemonik_mcp_payment_mode: "none" +mnemonik_mcp_rust_log: "info" +# Origin advertised in OAuth metadata + challenge envelopes. MUST match the +# public hostname external clients (Cursor/Claude.ai) connect to — RFC 8707 +# origin check is enforced. No trailing slash. +mnemonik_mcp_public_base_url: "https://mcp.mnemonik.xyz" +# Port the mcp server listens on inside the container. Caddy reverse-proxies +# to this over the shared docker network; NOT published to a host port. +mnemonik_mcp_container_port: 3000 + +# --- Caddy vhost (public, real Let's Encrypt) ------------------------------ +# Public hostname served on 443. Requires: DNS A record → VM public IPv4, +# firewall 80+443 open (already the case), and caddy_acme_email set so LE +# can issue. The webapp lives on Cloudflare Pages — this is the MCP host only. +mnemonik_public_domain: "mcp.mnemonik.xyz" + +# Shared Caddy — cross-role contract with the vaultwarden role. Snippet is +# dropped into its conf.d; the container is restarted to pick it up. +vaultwarden_caddy_container_name: "vaultwarden-caddy-1" +mnemonik_caddy_conf_d: "/opt/vaultwarden/caddy_conf_d" + +# External docker network created by the vaultwarden role's compose stack. +# The mcp container joins it so Caddy can reverse_proxy by name. +mnemonik_shared_caddy_network: "vaultwarden_vaultwarden" + +# --- Swap (ollama + fastembed headroom) ------------------------------------ +# Safety cushion recommended by the monorepo bootstrap. Idempotent. +mnemonik_server_swap_enabled: true +mnemonik_server_swap_size_mb: 4096 +mnemonik_server_swap_path: "/swapfile" + +# Health probe budget (post-up). MCP start_period is 180s (fastembed model +# load), so give it generous retries. +mnemonik_mcp_health_retries: 120 +mnemonik_mcp_health_delay: 3 + +# CRITICAL: provided by the playbook via sops decryption (deploy.yml). +# Not defined here so the role fails loudly if enabled without them. +# mnemonik_mcp_jwt_secret — openssl rand -base64 32, stable across deploys +# mnemonik_mcp_refresh_salt — openssl rand -base64 32, rotating invalidates +# every live refresh token (deliberate event) diff --git a/infrastructure/ansible/roles/mnemonik-server/handlers/main.yml b/infrastructure/ansible/roles/mnemonik-server/handlers/main.yml new file mode 100644 index 0000000..c073d90 --- /dev/null +++ b/infrastructure/ansible/roles/mnemonik-server/handlers/main.yml @@ -0,0 +1,22 @@ +--- +- name: Restart mnemonik-server + # community.docker.docker_compose_v2 lacks `restarted:` on the runner's + # collection (see kaneo handler note). Shell out; handler only fires when + # a rendered template actually changed, so idempotency isn't needed here. + ansible.builtin.command: + cmd: docker compose restart + chdir: "{{ mnemonik_server_install_dir }}" + environment: + COMPOSE_PROJECT_NAME: "{{ mnemonik_server_compose_project }}" + become: true + become_user: op + changed_when: true + +- name: Reload caddy for mnemonik vhost + # `docker restart` of the shared Caddy — the same reliable hammer the kaneo + # role uses (admin-API reload has flaked on this stack). Caddy re-reads + # Caddyfile + conf.d/*.conf on boot and issues LE certs for new vhosts. + become: true + ansible.builtin.command: + cmd: "docker restart {{ vaultwarden_caddy_container_name }}" + changed_when: true diff --git a/infrastructure/ansible/roles/mnemonik-server/meta/main.yml b/infrastructure/ansible/roles/mnemonik-server/meta/main.yml new file mode 100644 index 0000000..49b6f99 --- /dev/null +++ b/infrastructure/ansible/roles/mnemonik-server/meta/main.yml @@ -0,0 +1,17 @@ +--- +galaxy_info: + author: "Mnemonic Protocol" + description: "Deploy the Mnemonic MCP server (Rust GHCR image) + co-located Ollama, fronted by the shared Caddy" + company: "Mnemonic Protocol" + license: "MIT" + min_ansible_version: "2.15" + tags: + - docker + - mcp + - mnemonik + - ollama + - caddy + +dependencies: [] +# Ordering enforced by the playbook (deploy.yml roles list): must run after +# vaultwarden (owns the shared Caddy + external network) and kaneo. diff --git a/infrastructure/ansible/roles/mnemonik-server/tasks/main.yml b/infrastructure/ansible/roles/mnemonik-server/tasks/main.yml new file mode 100644 index 0000000..bd0007f --- /dev/null +++ b/infrastructure/ansible/roles/mnemonik-server/tasks/main.yml @@ -0,0 +1,231 @@ +--- +# mnemonik-server: co-located Mnemonic MCP server (Rust image from GHCR) + +# local Ollama, fronted by the shared Caddy. See defaults/main.yml for the +# upstream-adaptation notes. + +- name: Skip mnemonik-server (disabled) + ansible.builtin.debug: + msg: >- + mnemonik_server_enabled is false — skipping. Enable via group_vars or + -e mnemonik_server_enabled=true once DNS + sops secrets are in place. + when: not (mnemonik_server_enabled | default(false) | bool) + +- name: Deploy mnemonik-server + when: mnemonik_server_enabled | default(false) | bool + block: + - name: Assert required MCP secrets are present + ansible.builtin.assert: + that: + - mnemonik_mcp_jwt_secret is defined and mnemonik_mcp_jwt_secret | string | length >= 16 + - mnemonik_mcp_refresh_salt is defined and mnemonik_mcp_refresh_salt | string | length >= 16 + fail_msg: | + FATAL: Mnemonic MCP hosted-mode secrets missing. Add to sops and + wire through deploy.yml (vault_mnemonik_mcp_*). Generate each once + and keep stable across deploys: + mnemonik_mcp_jwt_secret (openssl rand -base64 32) + mnemonik_mcp_refresh_salt (openssl rand -base64 32) + Rotating MCP_REFRESH_SALT invalidates every live refresh token. + no_log: true + + # --- Swap: ollama + fastembed headroom --------------------------------- + - name: Configure swap file + when: mnemonik_server_swap_enabled | default(true) | bool + become: true + block: + - name: Check whether swap file already exists + ansible.builtin.stat: + path: "{{ mnemonik_server_swap_path }}" + register: mnemonik_swap_stat + + - name: Allocate swap file + ansible.builtin.command: + cmd: "fallocate -l {{ mnemonik_server_swap_size_mb }}M {{ mnemonik_server_swap_path }}" + when: not mnemonik_swap_stat.stat.exists + changed_when: true + register: mnemonik_swap_alloc + + - name: Secure swap file permissions + ansible.builtin.file: + path: "{{ mnemonik_server_swap_path }}" + owner: root + group: root + mode: "0600" + when: not mnemonik_swap_stat.stat.exists + + - name: Format swap file + ansible.builtin.command: + cmd: "mkswap {{ mnemonik_server_swap_path }}" + when: not mnemonik_swap_stat.stat.exists + changed_when: true + + - name: Enable swap file (persist in fstab) + ansible.posix.mount: + path: none + src: "{{ mnemonik_server_swap_path }}" + fstype: swap + opts: sw + state: present + + - name: Activate swap now + ansible.builtin.command: + cmd: "swapon {{ mnemonik_server_swap_path }}" + when: not mnemonik_swap_stat.stat.exists + changed_when: true + + # --- State on the Hetzner persistent volume ---------------------------- + - name: Find Hetzner data volume mountpoint (block device /dev/sdb) + ansible.builtin.set_fact: + mnemonik_volume_mount: >- + {{ ansible_facts.mounts + | selectattr('device', 'equalto', '/dev/sdb') + | map(attribute='mount') + | first | default('') }} + + - name: Assert the data volume is mounted + ansible.builtin.assert: + that: + - mnemonik_volume_mount | length > 0 + fail_msg: | + /dev/sdb (Hetzner data volume) is not mounted. Mnemonic MCP server + identity (id.json) + attestation DB must live on the persistent + volume. Check hcloud_volume_attachment in tofu/hetzner/main.tf. + + - name: Set state dir on the persistent volume + ansible.builtin.set_fact: + mnemonik_state_dir: "{{ mnemonik_volume_mount }}/mnemonik" + + - name: Ensure state subdirs exist on the persistent volume + become: true + ansible.builtin.file: + path: "{{ mnemonik_state_dir }}/{{ item }}" + state: directory + owner: op + group: op + mode: "0700" + loop: + - data + - keypair + - ollama + + - name: Create install dir + become: true + ansible.builtin.file: + path: "{{ mnemonik_server_install_dir }}" + state: directory + owner: op + group: op + mode: "0750" + + # --- GHCR pull auth (only if a token was supplied) --------------------- + - name: Log in to GHCR (private package pull) + when: mnemonik_ghcr_token | default('') | length > 0 + become: true + become_user: op + ansible.builtin.command: + cmd: "docker login ghcr.io -u {{ mnemonik_ghcr_username }} --password-stdin" + stdin: "{{ mnemonik_ghcr_token }}" + changed_when: false + no_log: true + + # --- Render compose + env + Caddy vhost -------------------------------- + - name: Render docker-compose.yml + become: true + ansible.builtin.template: + src: docker-compose.yml.j2 + dest: "{{ mnemonik_server_install_dir }}/docker-compose.yml" + owner: op + group: op + mode: "0640" + register: mnemonik_compose_template + notify: Restart mnemonik-server + + - name: Render mcp.env + become: true + ansible.builtin.template: + src: mcp.env.j2 + dest: "{{ mnemonik_server_install_dir }}/mcp.env" + owner: op + group: op + mode: "0600" + no_log: true + register: mnemonik_env_template + notify: Restart mnemonik-server + + - name: Render Caddy vhost snippet into shared conf.d + become: true + ansible.builtin.template: + src: Caddyfile.snippet.j2 + dest: "{{ mnemonik_caddy_conf_d }}/mnemonik-mcp.conf" + owner: op + group: op + mode: "0644" + register: mnemonik_caddy_snippet + notify: Reload caddy for mnemonik vhost + + # --- Bring the stack up ------------------------------------------------ + - name: Run docker compose to start the mnemonik stack + community.docker.docker_compose_v2: + project_src: "{{ mnemonik_server_install_dir }}" + state: present + pull: always + environment: + COMPOSE_PROJECT_NAME: "{{ mnemonik_server_compose_project }}" + become: true + become_user: op + register: mnemonik_compose_result + + - name: Wait for Ollama to accept commands + become: true + become_user: op + ansible.builtin.command: + cmd: "docker exec mnemonik-ollama-1 ollama list" + register: mnemonik_ollama_ready + until: mnemonik_ollama_ready.rc == 0 + retries: 30 + delay: 2 + changed_when: false + + - name: Pull the Ollama model (idempotent — no-op if already present) + become: true + become_user: op + ansible.builtin.command: + cmd: "docker exec mnemonik-ollama-1 ollama pull {{ mnemonik_ollama_model }}" + register: mnemonik_ollama_pull + changed_when: "'pulling' in (mnemonik_ollama_pull.stdout | default('') + mnemonik_ollama_pull.stderr | default(''))" + + - name: Warm up the Ollama model (non-fatal) + become: true + become_user: op + ansible.builtin.command: + cmd: >- + docker exec mnemonik-ollama-1 sh -c + 'ollama run {{ mnemonik_ollama_model }} hello >/dev/null 2>&1 || true' + changed_when: false + failed_when: false + + # --- Caddy reload must precede the public health probe ----------------- + - name: Flush handlers so Caddy reload runs before the health probe + ansible.builtin.meta: flush_handlers + + - name: Wait for MCP /health via Caddy (public TLS) + # Probes the real public hostname the same way an external client would. + # start_period is 180s (fastembed load) so allow a generous window. + ansible.builtin.uri: + url: "https://{{ mnemonik_public_domain }}/health" + method: GET + status_code: 200 + register: mnemonik_health + until: mnemonik_health.status == 200 + retries: "{{ mnemonik_mcp_health_retries }}" + delay: "{{ mnemonik_mcp_health_delay }}" + changed_when: false + + - name: Fail if MCP health check exceeded timeout + ansible.builtin.fail: + msg: | + Mnemonic MCP health check failed. Inspect on the VM: + docker compose -f {{ mnemonik_server_install_dir }}/docker-compose.yml logs --tail=200 + docker compose -f {{ mnemonik_server_install_dir }}/docker-compose.yml ps + Also confirm DNS ({{ mnemonik_public_domain }} → VM IP) and that + Caddy obtained a Let's Encrypt cert (caddy_acme_email set). + when: mnemonik_health.status is not defined or mnemonik_health.status != 200 diff --git a/infrastructure/ansible/roles/mnemonik-server/templates/Caddyfile.snippet.j2 b/infrastructure/ansible/roles/mnemonik-server/templates/Caddyfile.snippet.j2 new file mode 100644 index 0000000..7f43694 --- /dev/null +++ b/infrastructure/ansible/roles/mnemonik-server/templates/Caddyfile.snippet.j2 @@ -0,0 +1,20 @@ +# Public ingress for the Mnemonic MCP server. Real Let's Encrypt cert obtained +# automatically via Caddy's ACME on first request. Requires: +# - DNS A record {{ mnemonik_public_domain }} → VM public IPv4 +# - Hetzner firewall opens TCP 80 + 443 from 0.0.0.0/0 (main.tf — already so) +# - caddy_acme_email role var set (Let's Encrypt contact) +# The mcp container joins the shared caddy network with alias `mnemonik-mcp` +# (see the mnemonik-server docker-compose template). +{{ mnemonik_public_domain }} { + encode gzip + reverse_proxy http://mnemonik-mcp:{{ mnemonik_mcp_container_port }} { + header_up X-Real-IP {remote_host} + header_up X-Forwarded-For {remote_host} + header_up X-Forwarded-Proto {scheme} + } + + header Strict-Transport-Security "max-age=31536000; includeSubDomains" + header X-Content-Type-Options "nosniff" + header X-Frame-Options "SAMEORIGIN" + header Referrer-Policy "same-origin" +} diff --git a/infrastructure/ansible/roles/mnemonik-server/templates/docker-compose.yml.j2 b/infrastructure/ansible/roles/mnemonik-server/templates/docker-compose.yml.j2 new file mode 100644 index 0000000..3fec31c --- /dev/null +++ b/infrastructure/ansible/roles/mnemonik-server/templates/docker-compose.yml.j2 @@ -0,0 +1,79 @@ +# Mnemonic MCP server stack (co-located on the fabric VM). +# Rendered by the mnemonik-server role — DO NOT edit on the box; edit the +# template. Adapted from mnemonik-xyz/monorepo docker-compose.yml: the `mcp` +# and `ollama` services are kept; `nginx` + `certbot` are dropped because the +# shared Caddy (vaultwarden role) terminates TLS and reverse-proxies instead. +services: + mcp: + image: "{{ mnemonik_mcp_image }}:{{ mnemonik_mcp_image_tag }}" + container_name: mnemonik-mcp-1 + restart: unless-stopped + env_file: + - mcp.env + environment: + OLLAMA_URL: "http://ollama:11434" + depends_on: + ollama: + condition: service_healthy + required: false + # No host port publish: Caddy reaches this over the shared network by the + # `mnemonik-mcp` alias below. The server still listens on 0.0.0.0:{{ mnemonik_mcp_container_port }} + # inside the container. + expose: + - "{{ mnemonik_mcp_container_port }}" + volumes: + # State on the Hetzner persistent volume — server identity (id.json) + # and attestation DB survive a VM rebuild. + - "{{ mnemonik_state_dir }}/data:/data" + - "{{ mnemonik_state_dir }}/keypair:/keypair" + networks: + # Internal — mcp <-> ollama only. + mnemonik: {} + # Shared external network (vaultwarden role) so Caddy can route + # mcp.mnemonik.xyz to this container. Without the alias, Caddy fails + # with "mnemonik-mcp: no such host" on the reverse_proxy directive. + "{{ mnemonik_shared_caddy_network }}": + aliases: + - mnemonik-mcp + security_opt: + - no-new-privileges:true + tmpfs: + - /tmp + healthcheck: + test: ["CMD", "curl", "-fsS", "http://localhost:{{ mnemonik_mcp_container_port }}/health"] + interval: 30s + timeout: 5s + start_period: 180s + retries: 5 + logging: + driver: json-file + options: { max-size: "10m", max-file: "5" } + + ollama: + image: "{{ mnemonik_ollama_image }}" + container_name: mnemonik-ollama-1 + restart: unless-stopped + volumes: + # Model cache on the persistent volume — {{ mnemonik_ollama_model }} is + # pulled once (post-up task) and survives redeploys / rebuilds. + - "{{ mnemonik_state_dir }}/ollama:/root/.ollama" + networks: + - mnemonik + healthcheck: + # Stock ollama image has no curl; `ollama list` exits 0 once the server + # is up (connects to the local :11434 API). + test: ["CMD", "ollama", "list"] + interval: 30s + timeout: 5s + start_period: 60s + retries: 5 + logging: + driver: json-file + options: { max-size: "10m", max-file: "3" } + +networks: + mnemonik: + name: mnemonik + driver: bridge + "{{ mnemonik_shared_caddy_network }}": + external: true diff --git a/infrastructure/ansible/roles/mnemonik-server/templates/mcp.env.j2 b/infrastructure/ansible/roles/mnemonik-server/templates/mcp.env.j2 new file mode 100644 index 0000000..8ced2ef --- /dev/null +++ b/infrastructure/ansible/roles/mnemonik-server/templates/mcp.env.j2 @@ -0,0 +1,23 @@ +# Mnemonic MCP runtime env — rendered by the mnemonik-server role. +# DO NOT edit on the box; edit templates/mcp.env.j2. Mirrors the contract in +# mnemonik-xyz/monorepo mcp/deploy/compose.env.example, minus the nginx/certbot +# TLS knobs (Caddy owns TLS here). + +# Topology: co-located ollama on the internal network. +OLLAMA_URL=http://ollama:11434 +OLLAMA_MODEL={{ mnemonik_ollama_model }} + +# Server runtime. +STORAGE_MODE={{ mnemonik_mcp_storage_mode }} +EMBED_PROVIDER={{ mnemonik_mcp_embed_provider }} +TURBO_BITS={{ mnemonik_mcp_turbo_bits }} +PAYMENT_MODE={{ mnemonik_mcp_payment_mode }} +RUST_LOG={{ mnemonik_mcp_rust_log }} + +# Public origin advertised in OAuth metadata + challenge envelopes (RFC 8707 +# origin check). MUST match the hostname external clients connect to. +MCP_PUBLIC_BASE_URL={{ mnemonik_mcp_public_base_url }} + +# Hosted-mode secrets (from sops; generate once, keep stable across deploys). +MCP_JWT_SECRET={{ mnemonik_mcp_jwt_secret }} +MCP_REFRESH_SALT={{ mnemonik_mcp_refresh_salt }} diff --git a/infrastructure/ansible/roles/mnemonik-server/tests/test_role_contract.py b/infrastructure/ansible/roles/mnemonik-server/tests/test_role_contract.py new file mode 100644 index 0000000..76e37c3 --- /dev/null +++ b/infrastructure/ansible/roles/mnemonik-server/tests/test_role_contract.py @@ -0,0 +1,100 @@ +"""Contract tests for the mnemonik-server role. + +No live VM, no docker: assert the structural invariants that make this role +correct and distinct from the client-binary `mnemonic-mcp` role. + +Run: pytest infrastructure/ansible/roles/mnemonik-server/tests/ -v +""" +from pathlib import Path + +import yaml + +ROLE = Path(__file__).resolve().parents[1] + +# Minimal context to render the compose template for structural inspection. +_CTX = { + "mnemonik_mcp_image": "ghcr.io/mnemonik-xyz/mnemonic-mcp", + "mnemonik_mcp_image_tag": "latest", + "mnemonik_mcp_container_port": 3000, + "mnemonik_state_dir": "/mnt/vol/mnemonik", + "mnemonik_shared_caddy_network": "vaultwarden_vaultwarden", + "mnemonik_ollama_image": "ollama/ollama:0.21.2", + "mnemonik_ollama_model": "qwen2.5:3b", +} + + +def _load(rel): + return yaml.safe_load((ROLE / rel).read_text()) + + +def _text(rel): + return (ROLE / rel).read_text() + + +def _rendered_compose(): + from jinja2 import Environment + + raw = _text("templates/docker-compose.yml.j2") + return yaml.safe_load(Environment().from_string(raw).render(**_CTX)) + + +def test_disabled_by_default(): + # Master switch must default false — enabling is an explicit operator act. + assert _load("defaults/main.yml")["mnemonik_server_enabled"] is False + + +def test_pulls_image_never_builds_rust(): + # The box pulls the GHCR image; no service builds from a Dockerfile. + assert "ghcr.io/mnemonik-xyz/mnemonic-mcp" in _load("defaults/main.yml")["mnemonik_mcp_image"] + for name, svc in _rendered_compose()["services"].items(): + assert "build" not in svc, f"{name} must not build locally" + + +def test_drops_nginx_and_certbot(): + # Caddy fronts TLS; only mcp + ollama remain, and no host 80/443 bind. + compose = _rendered_compose() + assert set(compose["services"]) == {"mcp", "ollama"} + for svc in compose["services"].values(): + assert not svc.get("ports"), "no host port publish — Caddy fronts it" + + +def test_joins_shared_caddy_network_with_alias(): + compose = _rendered_compose() + mcp_nets = compose["services"]["mcp"]["networks"] + assert "vaultwarden_vaultwarden" in mcp_nets + assert "mnemonik-mcp" in mcp_nets["vaultwarden_vaultwarden"]["aliases"] + snippet = _text("templates/Caddyfile.snippet.j2") + assert "reverse_proxy http://mnemonik-mcp:" in snippet + + +def test_state_on_persistent_volume(): + # Identity (/keypair) + attestation DB (/data) bind-mounted off the volume. + volumes = _rendered_compose()["services"]["mcp"]["volumes"] + assert "/mnt/vol/mnemonik/keypair:/keypair" in volumes + assert "/mnt/vol/mnemonik/data:/data" in volumes + assert "/dev/sdb" in _text("tasks/main.yml") # discovers the volume mount + + +def test_secrets_asserted_and_have_no_default(): + # Fails loudly if enabled without the hosted-mode secrets. + defaults = _load("defaults/main.yml") + assert "mnemonik_mcp_jwt_secret" not in defaults + assert "mnemonik_mcp_refresh_salt" not in defaults + tasks = _text("tasks/main.yml") + assert "Assert required MCP secrets are present" in tasks + + +def test_ollama_model_pulled_not_built(): + # Stock ollama image + a post-up pull replaces the monorepo custom image. + compose = _text("templates/docker-compose.yml.j2") + assert "ollama/ollama" in _load("defaults/main.yml")["mnemonik_ollama_image"] + tasks = _text("tasks/main.yml") + assert "ollama pull" in tasks + + +def test_distinct_from_client_binary_role(): + # Guard against confusion with the `mnemonic-mcp` client-binary role: + # this role must not install an npm package or a PATH binary. + tasks = _text("tasks/main.yml") + assert "npm" not in tasks + assert "/usr/local/bin" not in tasks diff --git a/infrastructure/secrets/secrets.sops.yml.template b/infrastructure/secrets/secrets.sops.yml.template index bfbd3d7..6d71b2e 100644 --- a/infrastructure/secrets/secrets.sops.yml.template +++ b/infrastructure/secrets/secrets.sops.yml.template @@ -205,3 +205,14 @@ claude_blog_repo_ref: "" # sops to "" (drop trailing comma + old key), (6) redeploy. # Generate: openssl rand -hex 32 publish_callback_hmac_secrets: "" + +# --------------------------------------------------------------------------- +# Mnemonic MCP server (mnemonik-server role) — hosted/OAuth mode secrets. +# Consumed only when mnemonik_server_enabled=true. Generate each ONCE and keep +# stable across deploys; both feed the mcp container's env (MCP_JWT_SECRET / +# MCP_REFRESH_SALT). Rotating mnemonik_mcp_refresh_salt changes the at-rest +# refresh-token hash and invalidates every live refresh token — treat as a +# deliberate operational event, not a routine deploy. +# Generate: openssl rand -base64 32 +mnemonik_mcp_jwt_secret: "" +mnemonik_mcp_refresh_salt: "" From 51116b30c47c1df8efbc5638633e2eae482479c0 Mon Sep 17 00:00:00 2001 From: mnemonik-dev Date: Thu, 9 Jul 2026 02:36:36 +0300 Subject: [PATCH 02/19] chore(secrets): add mnemonik MCP hosted-mode secrets (jwt + refresh salt) --- infrastructure/secrets/secrets.sops.yml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/infrastructure/secrets/secrets.sops.yml b/infrastructure/secrets/secrets.sops.yml index b502397..f2b23ad 100644 --- a/infrastructure/secrets/secrets.sops.yml +++ b/infrastructure/secrets/secrets.sops.yml @@ -98,6 +98,10 @@ publish_callback_hmac_secrets: ENC[AES256_GCM,data:3/lqBMZMbLndWOWhIdyq6i43J3mX0 publish_auto_mode_token: ENC[AES256_GCM,data:2n+CCny2ifjVZPgGQaPDEAPVjn6y4WNMIhrCVRpUcEaoPq91,iv:CPl6SdgiuKK53V2BgTlmlvUuJTlzmrz8eyog5pnqXqQ=,tag:izQgpLQYf+XaCOaiF51liQ==,type:str] publish_auto_mode_token_confirm: ENC[AES256_GCM,data:RAnzHU+2/fPzxESbtoVHhWpdju1goUdN0zbO2cpGuKzJXTYR,iv:4Qo2G1Q/ZT+G5/tbYUQbOMbQk/8VNH2JxOsWcv1HFB8=,tag:mO682iGD9+qfZfs5MH/avA==,type:str] blogger_prompts_topic_id: ENC[AES256_GCM,data:NA==,iv:4kcqGHUtErP6MxGkz2B8Gnva9KumFs3YTlQDwUcs/no=,tag:1FWbKpUgJfmD5ZbctocoWg==,type:str] +#ENC[AES256_GCM,data:FurYZcZgmzkkb59aSqRBlbAS5w==,iv:uBvcuc+AlHbMZQb1kotp8D+bzuOy/h3LE1dicyL4KMU=,tag:+S+ykswDEj64SpJC0izuLw==,type:comment] +mnemonik_mcp_jwt_secret: ENC[AES256_GCM,data:qeHXvs7uxjurcQvHi9Z6sn8beJk5Uoj4AUtXs1RF9ImKW7eH3pgifewcAHY=,iv:kmn3TUwirpn5om7u7adtOh2Hj4ed8i3Mah850E6qprQ=,tag:7xtP+5nZddzCzFMd5Vu+yg==,type:str] +#ENC[AES256_GCM,data:qFXS5ftMSM7U0oB2i+kZk/jt1fkW,iv:U4aaKhbshMdAOCEkH35LgKw6jHCuscRaXWfcWrYCpNs=,tag:Pvx8LD87FJOB6ZUPGfNing==,type:comment] +mnemonik_mcp_refresh_salt: ENC[AES256_GCM,data:UMFormFhcpTn+XZaVXhmW9i3ROqUrpJZgEAPBjr3ehzZpkhB+p1c25rn5is=,iv:u2izlNzHb9E36Lb7ZQwoCSmzCg+ipZuPDtOF8Vm/TNQ=,tag:+EUZK5MB6Uv4NvJ+hiqlUg==,type:str] sops: age: - enc: | @@ -109,7 +113,7 @@ sops: mwML67TqItIVEPO/2zj+r9prghdPBD4qQuVo1SBJtyFRUrjGfLfTsQ== -----END AGE ENCRYPTED FILE----- recipient: age1wwf8jfxffrqzdd9ntxsry4c3n2rddta036fj5nlu0qhacxmeu52sj3u3yg - lastmodified: "2026-06-10T20:23:03Z" - mac: ENC[AES256_GCM,data:V00pvgllcJ7zhqIvi7p/KMWK1+id6ThKgkbsxh/pIJdrJG8BcPhQ5zPMLBdMV9Yg1S3mL29cQK++0hJOSr1bxbCpWWxvGuLel96oIg3jIpn87eKx6SqtztRdGNq9e31xPFL3id42T5+0ekFG825ld7aZCySOBy11eWEdRBmMiWM=,iv:UBdCgo4GUa17eC9U6zJs0tzKiu8ZdfB2Ixfoo/NPZEU=,tag:iOPhGtTxrKV9a2EDAPL3HQ==,type:str] + lastmodified: "2026-07-08T23:35:14Z" + mac: ENC[AES256_GCM,data:XL4arHwws1Z6uqMKjC0wBK89cNYaTifkauQbozF7mxUWGxRZjDx8qvHr0w59mK9NAOsICyAEL/tHYFZ+OIQjhAFxcZBFLbr/H2XfD4sBtPljBlUGbloFP7fJ/9qeaLZMCBbunsnXKR8/RChXpijm3tgQKghZYgJJDRd9YM3WVj0=,iv:MXEpUnw+OhKoKsa6YwmV/lOrSA/SEBaLuNY/2f3oVwU=,tag:sjlURTEUI4189PCF5BL+dA==,type:str] unencrypted_suffix: _unencrypted version: 3.13.0 From a9180fc37a24dd7fefb381974bc7dd6c51670a0c Mon Sep 17 00:00:00 2001 From: mnemonik-dev Date: Thu, 9 Jul 2026 02:37:36 +0300 Subject: [PATCH 03/19] fix(mnemonik-server): make GHCR login non-fatal so public-package pull works --- .../ansible/roles/mnemonik-server/tasks/main.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/infrastructure/ansible/roles/mnemonik-server/tasks/main.yml b/infrastructure/ansible/roles/mnemonik-server/tasks/main.yml index bd0007f..b5b34f5 100644 --- a/infrastructure/ansible/roles/mnemonik-server/tasks/main.yml +++ b/infrastructure/ansible/roles/mnemonik-server/tasks/main.yml @@ -117,7 +117,11 @@ mode: "0750" # --- GHCR pull auth (only if a token was supplied) --------------------- - - name: Log in to GHCR (private package pull) + # Non-fatal: if the package is PUBLIC, an empty/invalid token or a failed + # login must not abort the deploy — the subsequent `compose pull` still + # fetches a public image anonymously. Login only matters for a private + # package, and there a failure surfaces later as a pull error anyway. + - name: Log in to GHCR (private package pull; non-fatal) when: mnemonik_ghcr_token | default('') | length > 0 become: true become_user: op @@ -125,6 +129,7 @@ cmd: "docker login ghcr.io -u {{ mnemonik_ghcr_username }} --password-stdin" stdin: "{{ mnemonik_ghcr_token }}" changed_when: false + failed_when: false no_log: true # --- Render compose + env + Caddy vhost -------------------------------- From ca92eba733d43be1aad3b6c00f3027e750e92c92 Mon Sep 17 00:00:00 2001 From: mnemonik-dev Date: Thu, 9 Jul 2026 09:58:58 +0300 Subject: [PATCH 04/19] feat(deploy): enable mnemonik-server, pin image sha-61fdb3e --- .../ansible/inventory/group_vars/all/features.yml | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 infrastructure/ansible/inventory/group_vars/all/features.yml diff --git a/infrastructure/ansible/inventory/group_vars/all/features.yml b/infrastructure/ansible/inventory/group_vars/all/features.yml new file mode 100644 index 0000000..30ff9ef --- /dev/null +++ b/infrastructure/ansible/inventory/group_vars/all/features.yml @@ -0,0 +1,14 @@ +--- +# Feature enable flags, loaded by ansible from group_vars/all (adjacent to the +# CI-generated inventory hosts.yml). Higher precedence than role defaults, +# lower than the play's role vars in deploy.yml. + +# mnemonik-server: co-located Mnemonic MCP server (Rust GHCR image + local +# Ollama), fronted by the shared Caddy. Enabled 2026-07-09 once the GHCR +# package went public and a build passed. +mnemonik_server_enabled: true + +# Pin to the confirmed-good build rather than :latest for a reproducible +# deploy. Bump when a newer build is verified. Package is public, so the +# fail-soft GHCR login is skipped/ignored and the pull is anonymous. +mnemonik_mcp_image_tag: "sha-61fdb3e" From d1832841874b3fcb5d9b81c8be9537cd23f29d52 Mon Sep 17 00:00:00 2001 From: mnemonik-dev Date: Thu, 9 Jul 2026 10:59:35 +0300 Subject: [PATCH 05/19] fix(mnemonik-server): non-blocking ollama pull + health-fail diagnostics The synchronous ollama model pull held the SSH run until the deploy was killed (run 29000156212), aborting before the Caddy reload + health gate. Make it async/poll:0/failed_when:false (model not needed for MCP core; pulls lazily for /chat). Add docker ps + mnemonik-mcp-1 log capture on health failure since the CI runner can't be SSH'd post-run. --- .../roles/mnemonik-server/tasks/main.yml | 51 +++++++++++++------ 1 file changed, 35 insertions(+), 16 deletions(-) diff --git a/infrastructure/ansible/roles/mnemonik-server/tasks/main.yml b/infrastructure/ansible/roles/mnemonik-server/tasks/main.yml index b5b34f5..0ccd7c0 100644 --- a/infrastructure/ansible/roles/mnemonik-server/tasks/main.yml +++ b/infrastructure/ansible/roles/mnemonik-server/tasks/main.yml @@ -190,21 +190,19 @@ delay: 2 changed_when: false - - name: Pull the Ollama model (idempotent — no-op if already present) + # Fire-and-forget: pulling {{ mnemonik_ollama_model }} (~2GB) can take many + # minutes and MUST NOT block or fail the deploy — a synchronous pull here + # held the SSH run until it was killed (run 29000156212). The MCP server's + # /health + core tools do not need the model; only /chat does, and ollama + # serves it once the background pull finishes. async+poll:0 detaches it on + # the target so the play proceeds straight to the Caddy reload + health gate. + - name: Pull the Ollama model in the background (non-blocking, non-fatal) become: true become_user: op ansible.builtin.command: cmd: "docker exec mnemonik-ollama-1 ollama pull {{ mnemonik_ollama_model }}" - register: mnemonik_ollama_pull - changed_when: "'pulling' in (mnemonik_ollama_pull.stdout | default('') + mnemonik_ollama_pull.stderr | default(''))" - - - name: Warm up the Ollama model (non-fatal) - become: true - become_user: op - ansible.builtin.command: - cmd: >- - docker exec mnemonik-ollama-1 sh -c - 'ollama run {{ mnemonik_ollama_model }} hello >/dev/null 2>&1 || true' + async: 1800 + poll: 0 changed_when: false failed_when: false @@ -225,12 +223,33 @@ delay: "{{ mnemonik_mcp_health_delay }}" changed_when: false + # Dump container state + MCP logs into the CI log on failure — the deploy + # runner can't be SSH'd into afterward, so surface diagnostics inline. + - name: Capture MCP diagnostics on health failure + when: mnemonik_health.status is not defined or mnemonik_health.status != 200 + become: true + become_user: op + ansible.builtin.shell: | + echo "=== docker ps (mnemonik) ===" + docker ps -a --filter name=mnemonik + echo "=== mnemonik-mcp-1 logs (tail 100) ===" + docker logs mnemonik-mcp-1 --tail 100 2>&1 || true + echo "=== local curl to mcp:3000/health from host ===" + curl -sS -m 5 -o /dev/null -w 'container_health_http=%{http_code}\n' http://127.0.0.1:3000/health 2>&1 || echo "host-3000 not published (expected)" + register: mnemonik_diag + changed_when: false + failed_when: false + + - name: Show MCP diagnostics + when: mnemonik_health.status is not defined or mnemonik_health.status != 200 + ansible.builtin.debug: + var: mnemonik_diag.stdout_lines + - name: Fail if MCP health check exceeded timeout ansible.builtin.fail: msg: | - Mnemonic MCP health check failed. Inspect on the VM: - docker compose -f {{ mnemonik_server_install_dir }}/docker-compose.yml logs --tail=200 - docker compose -f {{ mnemonik_server_install_dir }}/docker-compose.yml ps - Also confirm DNS ({{ mnemonik_public_domain }} → VM IP) and that - Caddy obtained a Let's Encrypt cert (caddy_acme_email set). + Mnemonic MCP health check failed. See "Show MCP diagnostics" above for + container state + mnemonik-mcp-1 logs. Also confirm DNS + ({{ mnemonik_public_domain }} → VM IP) and that Caddy obtained a + Let's Encrypt cert (caddy_acme_email set). when: mnemonik_health.status is not defined or mnemonik_health.status != 200 From 5d4874d9a93dce07bce606fdca7036737dfc6958 Mon Sep 17 00:00:00 2001 From: mnemonik-dev Date: Thu, 9 Jul 2026 11:45:05 +0300 Subject: [PATCH 06/19] fix(mnemonik-server): loopback health probe + always-on diagnostics Probing the public hostname from the VM hairpins to its own public IP and hangs, stalling the SSH run until the CI job is cancelled (~15 min) with unreadable logs. Probe Caddy via --resolve :127.0.0.1 with -m 5 hard timeout, fast-fail (40x3s). Always dump docker ps + mnemonik-mcp-1 logs + a direct docker-exec /health so the 502 root cause is visible in the CI log. --- .../roles/mnemonik-server/tasks/main.yml | 57 ++++++++++--------- 1 file changed, 31 insertions(+), 26 deletions(-) diff --git a/infrastructure/ansible/roles/mnemonik-server/tasks/main.yml b/infrastructure/ansible/roles/mnemonik-server/tasks/main.yml index 0ccd7c0..ba67580 100644 --- a/infrastructure/ansible/roles/mnemonik-server/tasks/main.yml +++ b/infrastructure/ansible/roles/mnemonik-server/tasks/main.yml @@ -210,46 +210,51 @@ - name: Flush handlers so Caddy reload runs before the health probe ansible.builtin.meta: flush_handlers - - name: Wait for MCP /health via Caddy (public TLS) - # Probes the real public hostname the same way an external client would. - # start_period is 180s (fastembed load) so allow a generous window. - ansible.builtin.uri: - url: "https://{{ mnemonik_public_domain }}/health" - method: GET - status_code: 200 + - name: Probe MCP /health via Caddy on loopback (avoids hairpin NAT) + # Probing the PUBLIC hostname from the VM itself relies on hairpin NAT to + # the box's own public IP, which frequently hangs — every retry then + # stalls for the full timeout and the SSH run dies (~15 min), showing up + # as a cancelled CI job (runs 29000156212 / 29003276564). Resolve the + # vhost to 127.0.0.1 so we hit the local Caddy directly (real cert, SNI + # match), with a hard -m 5 per attempt. Fast-fail so the play completes + # and the diagnostics below are readable. + become: true + become_user: op + ansible.builtin.command: + cmd: >- + curl -fsS -m 5 --resolve {{ mnemonik_public_domain }}:443:127.0.0.1 + https://{{ mnemonik_public_domain }}/health register: mnemonik_health - until: mnemonik_health.status == 200 - retries: "{{ mnemonik_mcp_health_retries }}" - delay: "{{ mnemonik_mcp_health_delay }}" + until: mnemonik_health.rc == 0 + retries: 40 + delay: 3 changed_when: false + failed_when: false - # Dump container state + MCP logs into the CI log on failure — the deploy - # runner can't be SSH'd into afterward, so surface diagnostics inline. - - name: Capture MCP diagnostics on health failure - when: mnemonik_health.status is not defined or mnemonik_health.status != 200 + # Always capture container state + MCP logs + a direct-to-container health + # probe — the CI runner can't be SSH'd into afterward, so this inline dump + # is the only diagnostic channel. Cheap; invaluable when /health isn't 200. + - name: Capture MCP diagnostics become: true become_user: op ansible.builtin.shell: | echo "=== docker ps (mnemonik) ===" docker ps -a --filter name=mnemonik - echo "=== mnemonik-mcp-1 logs (tail 100) ===" - docker logs mnemonik-mcp-1 --tail 100 2>&1 || true - echo "=== local curl to mcp:3000/health from host ===" - curl -sS -m 5 -o /dev/null -w 'container_health_http=%{http_code}\n' http://127.0.0.1:3000/health 2>&1 || echo "host-3000 not published (expected)" + echo "=== mnemonik-mcp-1 direct /health (docker exec localhost:3000) ===" + docker exec mnemonik-mcp-1 curl -fsS -m 5 http://localhost:3000/health 2>&1; echo "(exec rc=$?)" + echo "=== mnemonik-mcp-1 logs (tail 120) ===" + docker logs mnemonik-mcp-1 --tail 120 2>&1 || true register: mnemonik_diag changed_when: false failed_when: false - name: Show MCP diagnostics - when: mnemonik_health.status is not defined or mnemonik_health.status != 200 ansible.builtin.debug: var: mnemonik_diag.stdout_lines - - name: Fail if MCP health check exceeded timeout + - name: Fail if MCP /health not 200 via Caddy ansible.builtin.fail: - msg: | - Mnemonic MCP health check failed. See "Show MCP diagnostics" above for - container state + mnemonik-mcp-1 logs. Also confirm DNS - ({{ mnemonik_public_domain }} → VM IP) and that Caddy obtained a - Let's Encrypt cert (caddy_acme_email set). - when: mnemonik_health.status is not defined or mnemonik_health.status != 200 + msg: >- + Mnemonic MCP /health did not return 200 via Caddy (loopback). See + "Show MCP diagnostics" above for container state + mnemonik-mcp-1 logs. + when: mnemonik_health.rc != 0 From d379d1ad281229049670ba0f9729780ecd1a1d1b Mon Sep 17 00:00:00 2001 From: mnemonik-dev Date: Thu, 9 Jul 2026 12:22:36 +0300 Subject: [PATCH 07/19] fix(mnemonik-server): pre-create /data/model-cache so fastembed can start The mcp image pre-creates /data/model-cache + /data/rag_chunks and downloads its fastembed ONNX model there on first boot. Bind-mounting the empty host /data masked those dirs, so fastembed couldn't write the model and the server FATALed ('No embedding provider available') in a restart loop -> Caddy 502 (run 29005805686 logs). Pre-create the subdirs (0777) on the host bind. --- .../roles/mnemonik-server/tasks/main.yml | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/infrastructure/ansible/roles/mnemonik-server/tasks/main.yml b/infrastructure/ansible/roles/mnemonik-server/tasks/main.yml index ba67580..acc7336 100644 --- a/infrastructure/ansible/roles/mnemonik-server/tasks/main.yml +++ b/infrastructure/ansible/roles/mnemonik-server/tasks/main.yml @@ -107,6 +107,25 @@ - keypair - ollama + # The mcp image pre-creates /data/model-cache + /data/rag_chunks and writes + # its first-boot fastembed model (~22MB, FASTEMBED_CACHE_DIR=/data/model-cache) + # and RAG seed there. Bind-mounting the empty host /data masks those dirs, + # so fastembed can't write the model and the server FATALs with "No + # embedding provider available" in a restart loop (run 29005805686). Create + # them on the host bind; 0777 sidesteps any container-user uid mismatch on + # these non-sensitive caches. + - name: Ensure MCP /data cache subdirs exist (fastembed model + RAG seed) + become: true + ansible.builtin.file: + path: "{{ mnemonik_state_dir }}/data/{{ item }}" + state: directory + owner: op + group: op + mode: "0777" + loop: + - model-cache + - rag_chunks + - name: Create install dir become: true ansible.builtin.file: From 9ad78dbb85e16b757a9399ccf23a0b07abead16f Mon Sep 17 00:00:00 2001 From: mnemonik-dev Date: Thu, 9 Jul 2026 13:04:56 +0300 Subject: [PATCH 08/19] diag(mnemonik-server): debug container probe for fastembed env/DNS/egress/cache --- .../ansible/roles/mnemonik-server/tasks/main.yml | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/infrastructure/ansible/roles/mnemonik-server/tasks/main.yml b/infrastructure/ansible/roles/mnemonik-server/tasks/main.yml index acc7336..a07aeff 100644 --- a/infrastructure/ansible/roles/mnemonik-server/tasks/main.yml +++ b/infrastructure/ansible/roles/mnemonik-server/tasks/main.yml @@ -263,6 +263,22 @@ docker exec mnemonik-mcp-1 curl -fsS -m 5 http://localhost:3000/health 2>&1; echo "(exec rc=$?)" echo "=== mnemonik-mcp-1 logs (tail 120) ===" docker logs mnemonik-mcp-1 --tail 120 2>&1 || true + echo "=== debug container: fastembed env + DNS + HF egress + cache dir ===" + docker rm -f mcp-dbg >/dev/null 2>&1 || true + docker run -d --name mcp-dbg --entrypoint sleep \ + --network mnemonik --env-file {{ mnemonik_server_install_dir }}/mcp.env \ + -v {{ mnemonik_state_dir }}/data:/data \ + {{ mnemonik_mcp_image }}:{{ mnemonik_mcp_image_tag }} 180 >/dev/null 2>&1 || echo "dbg-run-fail" + docker network connect vaultwarden_vaultwarden mcp-dbg >/dev/null 2>&1 || echo "dbg-net2-fail" + docker exec mcp-dbg sh -c ' + echo "-- env --"; env | grep -iE "embed|cache|hf_|^home=|onnx|openai" | sort + echo "-- whoami --"; id 2>&1 + echo "-- dns --"; getent hosts huggingface.co 2>&1 || echo dns-FAIL + echo "-- egress HF --"; curl -sS -m 10 -o /dev/null -w "hf=%{http_code}\n" https://huggingface.co 2>&1 || echo hf-FAIL + echo "-- egress HF CDN --"; curl -sS -m 10 -o /dev/null -w "cdn=%{http_code}\n" https://cdn-lfs.huggingface.co 2>&1 || echo cdn-FAIL + echo "-- cache dir --"; ls -la /data/model-cache 2>&1 | head + ' 2>&1 | head -40 + docker rm -f mcp-dbg >/dev/null 2>&1 || true register: mnemonik_diag changed_when: false failed_when: false From d7b2d1d931090d4270d977bf58f6d45488967ad9 Mon Sep 17 00:00:00 2001 From: mnemonik-dev Date: Thu, 9 Jul 2026 13:22:37 +0300 Subject: [PATCH 09/19] fix(mnemonik-server): chown state dirs to image uid 10001 so fastembed can start Debug probe (run 29010408334) proved: container runs as uid 10001/gid 999, HF egress works (hf=200), but /data (bind-mounted, owned by op/1000 mode 0700) was 'Permission denied' to the container -> fastembed couldn't access its cache -> FATAL restart loop -> 502. chown -R the persistent-volume /data + /keypair to 10001:999. Widen health gate to ~6min for first-boot model download + RAG seed. Remove the one-shot debug probe. --- .../roles/mnemonik-server/tasks/main.yml | 61 +++++++------------ 1 file changed, 23 insertions(+), 38 deletions(-) diff --git a/infrastructure/ansible/roles/mnemonik-server/tasks/main.yml b/infrastructure/ansible/roles/mnemonik-server/tasks/main.yml index a07aeff..be90fa7 100644 --- a/infrastructure/ansible/roles/mnemonik-server/tasks/main.yml +++ b/infrastructure/ansible/roles/mnemonik-server/tasks/main.yml @@ -95,36 +95,35 @@ mnemonik_state_dir: "{{ mnemonik_volume_mount }}/mnemonik" - name: Ensure state subdirs exist on the persistent volume + # /data holds the fastembed model cache (FASTEMBED_CACHE_DIR=/data/model-cache), + # RAG seed, and SQLite DB; /keypair holds the server identity. The image + # pre-creates /data/model-cache + /data/rag_chunks, but a bind mount masks + # them, so create them on the host too. become: true ansible.builtin.file: path: "{{ mnemonik_state_dir }}/{{ item }}" state: directory - owner: op - group: op - mode: "0700" + mode: "0755" loop: - data + - data/model-cache + - data/rag_chunks - keypair - ollama - # The mcp image pre-creates /data/model-cache + /data/rag_chunks and writes - # its first-boot fastembed model (~22MB, FASTEMBED_CACHE_DIR=/data/model-cache) - # and RAG seed there. Bind-mounting the empty host /data masks those dirs, - # so fastembed can't write the model and the server FATALs with "No - # embedding provider available" in a restart loop (run 29005805686). Create - # them on the host bind; 0777 sidesteps any container-user uid mismatch on - # these non-sensitive caches. - - name: Ensure MCP /data cache subdirs exist (fastembed model + RAG seed) + - name: Own MCP state dirs by the image's runtime uid (mnemonic = 10001:999) + # The mcp image runs as uid 10001 / gid 999. Bind-mounting host dirs owned + # by op(1000) made /data unreadable to the container ("Permission denied" + # on /data/model-cache, confirmed via debug probe run 29010408334), so + # fastembed FATALed with "No embedding provider available" in a restart + # loop. chown to the container's uid so it can read/write its model cache, + # identity, and DB. (The monorepo sidesteps this with named volumes, which + # inherit the image's ownership; we keep bind mounts for persistent-volume + # durability and set ownership explicitly.) become: true - ansible.builtin.file: - path: "{{ mnemonik_state_dir }}/data/{{ item }}" - state: directory - owner: op - group: op - mode: "0777" - loop: - - model-cache - - rag_chunks + ansible.builtin.command: + cmd: "chown -R 10001:999 {{ mnemonik_state_dir }}/data {{ mnemonik_state_dir }}/keypair" + changed_when: true - name: Create install dir become: true @@ -245,8 +244,10 @@ https://{{ mnemonik_public_domain }}/health register: mnemonik_health until: mnemonik_health.rc == 0 - retries: 40 - delay: 3 + # ~6 min: first boot downloads the ~22MB fastembed model + runs the RAG + # seed before /health goes 200 (image start_period is 180s). + retries: 90 + delay: 4 changed_when: false failed_when: false @@ -263,22 +264,6 @@ docker exec mnemonik-mcp-1 curl -fsS -m 5 http://localhost:3000/health 2>&1; echo "(exec rc=$?)" echo "=== mnemonik-mcp-1 logs (tail 120) ===" docker logs mnemonik-mcp-1 --tail 120 2>&1 || true - echo "=== debug container: fastembed env + DNS + HF egress + cache dir ===" - docker rm -f mcp-dbg >/dev/null 2>&1 || true - docker run -d --name mcp-dbg --entrypoint sleep \ - --network mnemonik --env-file {{ mnemonik_server_install_dir }}/mcp.env \ - -v {{ mnemonik_state_dir }}/data:/data \ - {{ mnemonik_mcp_image }}:{{ mnemonik_mcp_image_tag }} 180 >/dev/null 2>&1 || echo "dbg-run-fail" - docker network connect vaultwarden_vaultwarden mcp-dbg >/dev/null 2>&1 || echo "dbg-net2-fail" - docker exec mcp-dbg sh -c ' - echo "-- env --"; env | grep -iE "embed|cache|hf_|^home=|onnx|openai" | sort - echo "-- whoami --"; id 2>&1 - echo "-- dns --"; getent hosts huggingface.co 2>&1 || echo dns-FAIL - echo "-- egress HF --"; curl -sS -m 10 -o /dev/null -w "hf=%{http_code}\n" https://huggingface.co 2>&1 || echo hf-FAIL - echo "-- egress HF CDN --"; curl -sS -m 10 -o /dev/null -w "cdn=%{http_code}\n" https://cdn-lfs.huggingface.co 2>&1 || echo cdn-FAIL - echo "-- cache dir --"; ls -la /data/model-cache 2>&1 | head - ' 2>&1 | head -40 - docker rm -f mcp-dbg >/dev/null 2>&1 || true register: mnemonik_diag changed_when: false failed_when: false From 8303b775add8d95017f8120d3c86407e7dca3277 Mon Sep 17 00:00:00 2001 From: mnemonik-dev Date: Sat, 11 Jul 2026 16:17:21 +0300 Subject: [PATCH 10/19] feat(mnemonik-server): CORS for webapp, fastembed cache dir, chain-stats env, embedding provider check - Add optional mnemonik_mcp_cors_origin; Caddy emits preflight + simple CORS headers for the browser webapp at https://mnemonik.xyz. - Set FASTEMBED_CACHE_DIR so the embedding model download persists on the Hetzner volume across VM rebuilds. - Add CHAIN_STATS_WALLETS / SOLANA_RPC_URL env vars required by server images >= #194. - Verify fastembed initialized from container logs after deploy. - Fix role contract tests for CORS disabled-by-default check. --- .../inventory/group_vars/all/features.yml | 3 ++ .../roles/mnemonik-server/defaults/main.yml | 17 ++++++ .../roles/mnemonik-server/tasks/main.yml | 17 ++++++ .../templates/Caddyfile.snippet.j2 | 21 ++++++++ .../mnemonik-server/templates/mcp.env.j2 | 11 ++++ .../tests/test_role_contract.py | 52 +++++++++++++++++++ 6 files changed, 121 insertions(+) diff --git a/infrastructure/ansible/inventory/group_vars/all/features.yml b/infrastructure/ansible/inventory/group_vars/all/features.yml index 30ff9ef..ef77e93 100644 --- a/infrastructure/ansible/inventory/group_vars/all/features.yml +++ b/infrastructure/ansible/inventory/group_vars/all/features.yml @@ -12,3 +12,6 @@ mnemonik_server_enabled: true # deploy. Bump when a newer build is verified. Package is public, so the # fail-soft GHCR login is skipped/ignored and the pull is anonymous. mnemonik_mcp_image_tag: "sha-61fdb3e" + +# Browser webapp origin. Caddy emits CORS headers only for this exact origin. +mnemonik_mcp_cors_origin: "https://mnemonik.xyz" diff --git a/infrastructure/ansible/roles/mnemonik-server/defaults/main.yml b/infrastructure/ansible/roles/mnemonik-server/defaults/main.yml index 8382ab0..7aeeb5f 100644 --- a/infrastructure/ansible/roles/mnemonik-server/defaults/main.yml +++ b/infrastructure/ansible/roles/mnemonik-server/defaults/main.yml @@ -49,10 +49,27 @@ mnemonik_mcp_embed_provider: "fastembed" mnemonik_mcp_turbo_bits: 4 mnemonik_mcp_payment_mode: "none" mnemonik_mcp_rust_log: "info" + +# fastembed cache dir inside the container (bind-mounted from persistent +# volume via docker-compose.yml). Set explicitly so model downloads survive +# VM rebuilds and land in a writable, pre-created directory. +mnemonik_mcp_fastembed_cache_dir: "/data/model-cache" + +# Chain-backed traction stats (required by server images >= #194). +# Defaults are safe/public; override in sops/group_vars for real tracking. +mnemonik_chain_stats_wallets: "" +mnemonik_solana_rpc_url: "https://api.mainnet-beta.solana.com" # Origin advertised in OAuth metadata + challenge envelopes. MUST match the # public hostname external clients (Cursor/Claude.ai) connect to — RFC 8707 # origin check is enforced. No trailing slash. mnemonik_mcp_public_base_url: "https://mcp.mnemonik.xyz" + +# CORS origin for the webapp frontend. Set to the exact origin the browser +# sends (e.g. "https://mnemonik.xyz"). Empty string disables CORS headers at +# the Caddy layer — useful if the MCP server handles CORS itself or if the +# frontend is not yet deployed. Do NOT use "*" with credentials. +mnemonik_mcp_cors_origin: "" + # Port the mcp server listens on inside the container. Caddy reverse-proxies # to this over the shared docker network; NOT published to a host port. mnemonik_mcp_container_port: 3000 diff --git a/infrastructure/ansible/roles/mnemonik-server/tasks/main.yml b/infrastructure/ansible/roles/mnemonik-server/tasks/main.yml index be90fa7..44f1195 100644 --- a/infrastructure/ansible/roles/mnemonik-server/tasks/main.yml +++ b/infrastructure/ansible/roles/mnemonik-server/tasks/main.yml @@ -278,3 +278,20 @@ Mnemonic MCP /health did not return 200 via Caddy (loopback). See "Show MCP diagnostics" above for container state + mnemonik-mcp-1 logs. when: mnemonik_health.rc != 0 + + # /health only proves the HTTP surface is up; it does NOT prove the + # embedding provider loaded. A missing model cache or failed fastembed + # download leaves the server running but recall/semantic search broken. + # Fail loud here so the operator sees the real problem in CI logs. + - name: Verify fastembed embedding provider initialized + become: true + become_user: op + ansible.builtin.shell: | + set -euo pipefail + docker logs mnemonik-mcp-1 --tail 80 2>&1 | grep -qE 'Embedder: fastembed|fastembed \(.*verifiable=true\)' && exit 0 + echo "ERROR: fastembed embedding provider did not initialize in mnemonik-mcp-1" + echo "Check that FASTEMBED_CACHE_DIR is writable and the model can be downloaded." + exit 1 + register: mnemonik_embed_check + changed_when: false + failed_when: mnemonik_embed_check.rc != 0 diff --git a/infrastructure/ansible/roles/mnemonik-server/templates/Caddyfile.snippet.j2 b/infrastructure/ansible/roles/mnemonik-server/templates/Caddyfile.snippet.j2 index 7f43694..a14e879 100644 --- a/infrastructure/ansible/roles/mnemonik-server/templates/Caddyfile.snippet.j2 +++ b/infrastructure/ansible/roles/mnemonik-server/templates/Caddyfile.snippet.j2 @@ -7,6 +7,27 @@ # (see the mnemonik-server docker-compose template). {{ mnemonik_public_domain }} { encode gzip + +{% if mnemonik_mcp_cors_origin | default('') | length > 0 %} + # CORS for the browser-based webapp frontend. + # Preflight OPTIONS must short-circuit before reverse_proxy. + @cors_origin header Origin "{{ mnemonik_mcp_cors_origin }}" + @cors_preflight method OPTIONS + + handle @cors_preflight { + header Access-Control-Allow-Origin "{{ mnemonik_mcp_cors_origin }}" + header Access-Control-Allow-Methods "GET, POST, OPTIONS" + header Access-Control-Allow-Headers "Content-Type, Authorization" + header Vary "Origin" + respond 204 + } + + header @cors_origin Access-Control-Allow-Origin "{{ mnemonik_mcp_cors_origin }}" + header @cors_origin Access-Control-Allow-Methods "GET, POST, OPTIONS" + header @cors_origin Access-Control-Allow-Headers "Content-Type, Authorization" + header @cors_origin Vary "Origin" +{% endif %} + reverse_proxy http://mnemonik-mcp:{{ mnemonik_mcp_container_port }} { header_up X-Real-IP {remote_host} header_up X-Forwarded-For {remote_host} diff --git a/infrastructure/ansible/roles/mnemonik-server/templates/mcp.env.j2 b/infrastructure/ansible/roles/mnemonik-server/templates/mcp.env.j2 index 8ced2ef..35195f2 100644 --- a/infrastructure/ansible/roles/mnemonik-server/templates/mcp.env.j2 +++ b/infrastructure/ansible/roles/mnemonik-server/templates/mcp.env.j2 @@ -14,6 +14,12 @@ TURBO_BITS={{ mnemonik_mcp_turbo_bits }} PAYMENT_MODE={{ mnemonik_mcp_payment_mode }} RUST_LOG={{ mnemonik_mcp_rust_log }} +# fastembed model cache lives on the persistent volume (bind-mounted to +# /data in the container). Without this override the binary may try to +# download models to a per-user cache or fail to find them after a VM +# rebuild, which surfaces as "No embedding provider available". +FASTEMBED_CACHE_DIR={{ mnemonik_mcp_fastembed_cache_dir }} + # Public origin advertised in OAuth metadata + challenge envelopes (RFC 8707 # origin check). MUST match the hostname external clients connect to. MCP_PUBLIC_BASE_URL={{ mnemonik_mcp_public_base_url }} @@ -21,3 +27,8 @@ MCP_PUBLIC_BASE_URL={{ mnemonik_mcp_public_base_url }} # Hosted-mode secrets (from sops; generate once, keep stable across deploys). MCP_JWT_SECRET={{ mnemonik_mcp_jwt_secret }} MCP_REFRESH_SALT={{ mnemonik_mcp_refresh_salt }} + +# Chain-backed traction stats (required by server images >= #194). +# Safe defaults point to a public RPC and an empty watch wallet list. +CHAIN_STATS_WALLETS={{ mnemonik_chain_stats_wallets | default('') }} +SOLANA_RPC_URL={{ mnemonik_solana_rpc_url | default('https://api.mainnet-beta.solana.com') }} diff --git a/infrastructure/ansible/roles/mnemonik-server/tests/test_role_contract.py b/infrastructure/ansible/roles/mnemonik-server/tests/test_role_contract.py index 76e37c3..1cb6ad5 100644 --- a/infrastructure/ansible/roles/mnemonik-server/tests/test_role_contract.py +++ b/infrastructure/ansible/roles/mnemonik-server/tests/test_role_contract.py @@ -92,6 +92,58 @@ def test_ollama_model_pulled_not_built(): assert "ollama pull" in tasks +def test_fastembed_cache_dir_explicit(): + # FASTEMBED_CACHE_DIR must be set and point to the bind-mounted /data + # subdir so model downloads survive VM rebuilds. + env = _text("templates/mcp.env.j2") + assert "FASTEMBED_CACHE_DIR={{ mnemonik_mcp_fastembed_cache_dir }}" in env + defaults = _load("defaults/main.yml") + assert defaults["mnemonik_mcp_fastembed_cache_dir"] == "/data/model-cache" + + +def test_chain_stats_env_defaults(): + # Images >= #194 require these keys; defaults must be safe/public. + env = _text("templates/mcp.env.j2") + assert "CHAIN_STATS_WALLETS={{ mnemonik_chain_stats_wallets" in env + assert "SOLANA_RPC_URL={{ mnemonik_solana_rpc_url" in env + defaults = _load("defaults/main.yml") + assert defaults["mnemonik_solana_rpc_url"] == "https://api.mainnet-beta.solana.com" + + +def test_cors_disabled_by_default(): + # By default no CORS origin is configured; the snippet must NOT emit + # Access-Control-Allow-Origin, so we don't accidentally open the API. + from jinja2 import Environment + + raw = _text("templates/Caddyfile.snippet.j2") + rendered = Environment().from_string(raw).render( + mnemonik_public_domain="mcp.mnemonik.xyz", + mnemonik_mcp_container_port=3000, + mnemonik_mcp_cors_origin="", + ) + assert "Access-Control-Allow-Origin" not in rendered + defaults = _load("defaults/main.yml") + assert defaults["mnemonik_mcp_cors_origin"] == "" + + +def test_cors_enabled_when_origin_set(): + # Rendering with a webapp origin must produce the preflight + simple + # request CORS headers for that exact origin. + from jinja2 import Environment + + raw = _text("templates/Caddyfile.snippet.j2") + rendered = Environment().from_string(raw).render( + mnemonik_public_domain="mcp.mnemonik.xyz", + mnemonik_mcp_container_port=3000, + mnemonik_mcp_cors_origin="https://mnemonik.xyz", + ) + assert 'Access-Control-Allow-Origin "https://mnemonik.xyz"' in rendered + assert "Access-Control-Allow-Methods" in rendered + assert "Access-Control-Allow-Headers" in rendered + assert "@cors_preflight" in rendered + assert "respond 204" in rendered + + def test_distinct_from_client_binary_role(): # Guard against confusion with the `mnemonic-mcp` client-binary role: # this role must not install an npm package or a PATH binary. From 982f675797dfca8509559cd7137fc0473cfead94 Mon Sep 17 00:00:00 2001 From: mnemonik-dev Date: Sat, 11 Jul 2026 16:28:53 +0300 Subject: [PATCH 11/19] fix(mnemonik-server): use bash for pipefail in fastembed embed check --- infrastructure/ansible/roles/mnemonik-server/tasks/main.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/infrastructure/ansible/roles/mnemonik-server/tasks/main.yml b/infrastructure/ansible/roles/mnemonik-server/tasks/main.yml index 44f1195..0b9398a 100644 --- a/infrastructure/ansible/roles/mnemonik-server/tasks/main.yml +++ b/infrastructure/ansible/roles/mnemonik-server/tasks/main.yml @@ -292,6 +292,8 @@ echo "ERROR: fastembed embedding provider did not initialize in mnemonik-mcp-1" echo "Check that FASTEMBED_CACHE_DIR is writable and the model can be downloaded." exit 1 + args: + executable: /bin/bash register: mnemonik_embed_check changed_when: false failed_when: mnemonik_embed_check.rc != 0 From 12f905c5729fdc7edd3a7ef437dc727169518334 Mon Sep 17 00:00:00 2001 From: mnemonik-dev Date: Sat, 11 Jul 2026 16:40:36 +0300 Subject: [PATCH 12/19] fix(mnemonik-server): force-recreate on restart, longer embed log tail --- .../ansible/roles/mnemonik-server/handlers/main.yml | 5 ++++- infrastructure/ansible/roles/mnemonik-server/tasks/main.yml | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/infrastructure/ansible/roles/mnemonik-server/handlers/main.yml b/infrastructure/ansible/roles/mnemonik-server/handlers/main.yml index c073d90..05f8ff5 100644 --- a/infrastructure/ansible/roles/mnemonik-server/handlers/main.yml +++ b/infrastructure/ansible/roles/mnemonik-server/handlers/main.yml @@ -3,8 +3,11 @@ # community.docker.docker_compose_v2 lacks `restarted:` on the runner's # collection (see kaneo handler note). Shell out; handler only fires when # a rendered template actually changed, so idempotency isn't needed here. + # `--force-recreate` ensures env_file/compose changes (e.g. FASTEMBED_CACHE_DIR) + # are picked up; `docker compose restart` alone keeps the old env in the + # container. ansible.builtin.command: - cmd: docker compose restart + cmd: docker compose up -d --force-recreate chdir: "{{ mnemonik_server_install_dir }}" environment: COMPOSE_PROJECT_NAME: "{{ mnemonik_server_compose_project }}" diff --git a/infrastructure/ansible/roles/mnemonik-server/tasks/main.yml b/infrastructure/ansible/roles/mnemonik-server/tasks/main.yml index 0b9398a..c691e81 100644 --- a/infrastructure/ansible/roles/mnemonik-server/tasks/main.yml +++ b/infrastructure/ansible/roles/mnemonik-server/tasks/main.yml @@ -288,7 +288,7 @@ become_user: op ansible.builtin.shell: | set -euo pipefail - docker logs mnemonik-mcp-1 --tail 80 2>&1 | grep -qE 'Embedder: fastembed|fastembed \(.*verifiable=true\)' && exit 0 + docker logs mnemonik-mcp-1 --tail 500 2>&1 | grep -qE 'Embedder: fastembed|fastembed \(.*verifiable=true\)' && exit 0 echo "ERROR: fastembed embedding provider did not initialize in mnemonik-mcp-1" echo "Check that FASTEMBED_CACHE_DIR is writable and the model can be downloaded." exit 1 From cd7f4973b1c3cb34f0bc3474842b1621072536c2 Mon Sep 17 00:00:00 2001 From: mnemonik-dev Date: Sat, 11 Jul 2026 18:09:38 +0300 Subject: [PATCH 13/19] fix(mnemonik-server): always force-recreate mcp containers on deploy --- .../ansible/roles/mnemonik-server/tasks/main.yml | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/infrastructure/ansible/roles/mnemonik-server/tasks/main.yml b/infrastructure/ansible/roles/mnemonik-server/tasks/main.yml index c691e81..9f805fd 100644 --- a/infrastructure/ansible/roles/mnemonik-server/tasks/main.yml +++ b/infrastructure/ansible/roles/mnemonik-server/tasks/main.yml @@ -160,7 +160,6 @@ group: op mode: "0640" register: mnemonik_compose_template - notify: Restart mnemonik-server - name: Render mcp.env become: true @@ -172,7 +171,6 @@ mode: "0600" no_log: true register: mnemonik_env_template - notify: Restart mnemonik-server - name: Render Caddy vhost snippet into shared conf.d become: true @@ -187,14 +185,19 @@ # --- Bring the stack up ------------------------------------------------ - name: Run docker compose to start the mnemonik stack - community.docker.docker_compose_v2: - project_src: "{{ mnemonik_server_install_dir }}" - state: present - pull: always + # `--force-recreate` is required because Docker Compose does not recreate + # a container when only the env_file contents change; `restart` keeps the + # old env inside the running container. Always recreate on deploy so the + # MCP server picks up new env vars (FASTEMBED_CACHE_DIR, CORS origin, etc.) + # and the latest pinned image. + ansible.builtin.command: + cmd: docker compose up -d --pull always --force-recreate + chdir: "{{ mnemonik_server_install_dir }}" environment: COMPOSE_PROJECT_NAME: "{{ mnemonik_server_compose_project }}" become: true become_user: op + changed_when: true register: mnemonik_compose_result - name: Wait for Ollama to accept commands From a931c640250ea0ce84422616dbd5a7baaa567919 Mon Sep 17 00:00:00 2001 From: mnemonik-dev Date: Sat, 11 Jul 2026 21:04:23 +0300 Subject: [PATCH 14/19] fix(mnemonik-server): strip upstream CORS headers to avoid duplicates --- .../mnemonik-server/templates/Caddyfile.snippet.j2 | 10 ++++++++++ .../roles/mnemonik-server/tests/test_role_contract.py | 1 + 2 files changed, 11 insertions(+) diff --git a/infrastructure/ansible/roles/mnemonik-server/templates/Caddyfile.snippet.j2 b/infrastructure/ansible/roles/mnemonik-server/templates/Caddyfile.snippet.j2 index a14e879..dd35514 100644 --- a/infrastructure/ansible/roles/mnemonik-server/templates/Caddyfile.snippet.j2 +++ b/infrastructure/ansible/roles/mnemonik-server/templates/Caddyfile.snippet.j2 @@ -32,6 +32,16 @@ header_up X-Real-IP {remote_host} header_up X-Forwarded-For {remote_host} header_up X-Forwarded-Proto {scheme} + +{% if mnemonik_mcp_cors_origin | default('') | length > 0 %} + # The MCP server also emits CORS headers for its internal allowlist. Strip + # them so Caddy is the single source of truth and browsers see exactly one + # Access-Control-Allow-Origin value. + header_down -Access-Control-Allow-Origin + header_down -Access-Control-Allow-Methods + header_down -Access-Control-Allow-Headers + header_down -Vary +{% endif %} } header Strict-Transport-Security "max-age=31536000; includeSubDomains" diff --git a/infrastructure/ansible/roles/mnemonik-server/tests/test_role_contract.py b/infrastructure/ansible/roles/mnemonik-server/tests/test_role_contract.py index 1cb6ad5..4535538 100644 --- a/infrastructure/ansible/roles/mnemonik-server/tests/test_role_contract.py +++ b/infrastructure/ansible/roles/mnemonik-server/tests/test_role_contract.py @@ -142,6 +142,7 @@ def test_cors_enabled_when_origin_set(): assert "Access-Control-Allow-Headers" in rendered assert "@cors_preflight" in rendered assert "respond 204" in rendered + assert "header_down -Access-Control-Allow-Origin" in rendered def test_distinct_from_client_binary_role(): From 0f0df8f75405e30a60039b0f9aebd4b935b94609 Mon Sep 17 00:00:00 2001 From: mnemonik-dev Date: Sun, 12 Jul 2026 11:35:08 +0300 Subject: [PATCH 15/19] feat(mnemonik-server): support multiple CORS origins - mnemonik_mcp_cors_origin can now be a string or a list. - Caddy reflects the request Origin when it matches an allowed origin. - Update group_vars to allow www.mnemonik.xyz and mnemonik-webapp.pages.dev. --- .../inventory/group_vars/all/features.yml | 7 ++++-- .../roles/mnemonik-server/defaults/main.yml | 8 +++--- .../templates/Caddyfile.snippet.j2 | 18 +++++++++---- .../tests/test_role_contract.py | 25 ++++++++++++++++--- 4 files changed, 44 insertions(+), 14 deletions(-) diff --git a/infrastructure/ansible/inventory/group_vars/all/features.yml b/infrastructure/ansible/inventory/group_vars/all/features.yml index ef77e93..0a4892a 100644 --- a/infrastructure/ansible/inventory/group_vars/all/features.yml +++ b/infrastructure/ansible/inventory/group_vars/all/features.yml @@ -13,5 +13,8 @@ mnemonik_server_enabled: true # fail-soft GHCR login is skipped/ignored and the pull is anonymous. mnemonik_mcp_image_tag: "sha-61fdb3e" -# Browser webapp origin. Caddy emits CORS headers only for this exact origin. -mnemonik_mcp_cors_origin: "https://mnemonik.xyz" +# Browser webapp origins. Caddy reflects the request Origin if it matches +# one of these exact origins. +mnemonik_mcp_cors_origin: + - "https://www.mnemonik.xyz" + - "https://mnemonik-webapp.pages.dev" diff --git a/infrastructure/ansible/roles/mnemonik-server/defaults/main.yml b/infrastructure/ansible/roles/mnemonik-server/defaults/main.yml index 7aeeb5f..4adc390 100644 --- a/infrastructure/ansible/roles/mnemonik-server/defaults/main.yml +++ b/infrastructure/ansible/roles/mnemonik-server/defaults/main.yml @@ -64,10 +64,10 @@ mnemonik_solana_rpc_url: "https://api.mainnet-beta.solana.com" # origin check is enforced. No trailing slash. mnemonik_mcp_public_base_url: "https://mcp.mnemonik.xyz" -# CORS origin for the webapp frontend. Set to the exact origin the browser -# sends (e.g. "https://mnemonik.xyz"). Empty string disables CORS headers at -# the Caddy layer — useful if the MCP server handles CORS itself or if the -# frontend is not yet deployed. Do NOT use "*" with credentials. +# CORS origins for the webapp frontend. Set to the exact origin(s) the browser +# sends (e.g. "https://mnemonik.xyz" or ["https://www.mnemonik.xyz", +# "https://mnemonik-webapp.pages.dev"]). Empty string or empty list disables +# CORS headers at the Caddy layer. Do NOT use "*" with credentials. mnemonik_mcp_cors_origin: "" # Port the mcp server listens on inside the container. Caddy reverse-proxies diff --git a/infrastructure/ansible/roles/mnemonik-server/templates/Caddyfile.snippet.j2 b/infrastructure/ansible/roles/mnemonik-server/templates/Caddyfile.snippet.j2 index dd35514..e1f7ef2 100644 --- a/infrastructure/ansible/roles/mnemonik-server/templates/Caddyfile.snippet.j2 +++ b/infrastructure/ansible/roles/mnemonik-server/templates/Caddyfile.snippet.j2 @@ -5,24 +5,32 @@ # - caddy_acme_email role var set (Let's Encrypt contact) # The mcp container joins the shared caddy network with alias `mnemonik-mcp` # (see the mnemonik-server docker-compose template). +{% set _cors_origins = [] %} +{% if mnemonik_mcp_cors_origin is defined %} +{% if mnemonik_mcp_cors_origin is string and mnemonik_mcp_cors_origin | length > 0 %} +{% set _cors_origins = [mnemonik_mcp_cors_origin] %} +{% elif mnemonik_mcp_cors_origin is iterable and mnemonik_mcp_cors_origin is not string %} +{% set _cors_origins = mnemonik_mcp_cors_origin | list %} +{% endif %} +{% endif %} {{ mnemonik_public_domain }} { encode gzip -{% if mnemonik_mcp_cors_origin | default('') | length > 0 %} +{% if _cors_origins | length > 0 %} # CORS for the browser-based webapp frontend. # Preflight OPTIONS must short-circuit before reverse_proxy. - @cors_origin header Origin "{{ mnemonik_mcp_cors_origin }}" + @cors_origin header Origin{% for origin in _cors_origins %} "{{ origin }}"{% endfor %} @cors_preflight method OPTIONS handle @cors_preflight { - header Access-Control-Allow-Origin "{{ mnemonik_mcp_cors_origin }}" + header Access-Control-Allow-Origin "{http.request.header.Origin}" header Access-Control-Allow-Methods "GET, POST, OPTIONS" header Access-Control-Allow-Headers "Content-Type, Authorization" header Vary "Origin" respond 204 } - header @cors_origin Access-Control-Allow-Origin "{{ mnemonik_mcp_cors_origin }}" + header @cors_origin Access-Control-Allow-Origin "{http.request.header.Origin}" header @cors_origin Access-Control-Allow-Methods "GET, POST, OPTIONS" header @cors_origin Access-Control-Allow-Headers "Content-Type, Authorization" header @cors_origin Vary "Origin" @@ -33,7 +41,7 @@ header_up X-Forwarded-For {remote_host} header_up X-Forwarded-Proto {scheme} -{% if mnemonik_mcp_cors_origin | default('') | length > 0 %} +{% if _cors_origins | length > 0 %} # The MCP server also emits CORS headers for its internal allowlist. Strip # them so Caddy is the single source of truth and browsers see exactly one # Access-Control-Allow-Origin value. diff --git a/infrastructure/ansible/roles/mnemonik-server/tests/test_role_contract.py b/infrastructure/ansible/roles/mnemonik-server/tests/test_role_contract.py index 4535538..a3e4ad1 100644 --- a/infrastructure/ansible/roles/mnemonik-server/tests/test_role_contract.py +++ b/infrastructure/ansible/roles/mnemonik-server/tests/test_role_contract.py @@ -127,8 +127,8 @@ def test_cors_disabled_by_default(): def test_cors_enabled_when_origin_set(): - # Rendering with a webapp origin must produce the preflight + simple - # request CORS headers for that exact origin. + # Rendering with a single webapp origin must produce the preflight + simple + # request CORS headers and reflect the request Origin. from jinja2 import Environment raw = _text("templates/Caddyfile.snippet.j2") @@ -137,7 +137,8 @@ def test_cors_enabled_when_origin_set(): mnemonik_mcp_container_port=3000, mnemonik_mcp_cors_origin="https://mnemonik.xyz", ) - assert 'Access-Control-Allow-Origin "https://mnemonik.xyz"' in rendered + assert 'header Origin "https://mnemonik.xyz"' in rendered + assert 'Access-Control-Allow-Origin "{http.request.header.Origin}"' in rendered assert "Access-Control-Allow-Methods" in rendered assert "Access-Control-Allow-Headers" in rendered assert "@cors_preflight" in rendered @@ -145,6 +146,24 @@ def test_cors_enabled_when_origin_set(): assert "header_down -Access-Control-Allow-Origin" in rendered +def test_cors_enabled_with_multiple_origins(): + # A list of origins must be accepted and reflected. + from jinja2 import Environment + + raw = _text("templates/Caddyfile.snippet.j2") + rendered = Environment().from_string(raw).render( + mnemonik_public_domain="mcp.mnemonik.xyz", + mnemonik_mcp_container_port=3000, + mnemonik_mcp_cors_origin=[ + "https://www.mnemonik.xyz", + "https://mnemonik-webapp.pages.dev", + ], + ) + assert 'header Origin "https://www.mnemonik.xyz" "https://mnemonik-webapp.pages.dev"' in rendered + assert 'Access-Control-Allow-Origin "{http.request.header.Origin}"' in rendered + assert "@cors_preflight" in rendered + + def test_distinct_from_client_binary_role(): # Guard against confusion with the `mnemonic-mcp` client-binary role: # this role must not install an npm package or a PATH binary. From 62796658b622d65c9dfc53d56bc45a9db0340146 Mon Sep 17 00:00:00 2001 From: mnemonik-dev Date: Sun, 12 Jul 2026 12:00:59 +0300 Subject: [PATCH 16/19] fix(mnemonik-server): use Caddy expression matcher for multi-origin CORS Caddy's `header` matcher only accepts a single value, so the multi-origin render broke the shared Caddy config and left port 443 unbound. Switch to an expression matcher that ORs the allowed origins. Also capture Caddy logs + listening sockets in MCP diagnostics so a similar failure is obvious in CI. --- .../ansible/roles/mnemonik-server/tasks/main.yml | 6 ++++++ .../roles/mnemonik-server/templates/Caddyfile.snippet.j2 | 4 +++- .../roles/mnemonik-server/tests/test_role_contract.py | 8 ++++++-- 3 files changed, 15 insertions(+), 3 deletions(-) diff --git a/infrastructure/ansible/roles/mnemonik-server/tasks/main.yml b/infrastructure/ansible/roles/mnemonik-server/tasks/main.yml index 9f805fd..8b687e9 100644 --- a/infrastructure/ansible/roles/mnemonik-server/tasks/main.yml +++ b/infrastructure/ansible/roles/mnemonik-server/tasks/main.yml @@ -261,12 +261,18 @@ become: true become_user: op ansible.builtin.shell: | + echo "=== listening sockets ===" + ss -ltn 2>&1 || netstat -ltn 2>&1 || true echo "=== docker ps (mnemonik) ===" docker ps -a --filter name=mnemonik + echo "=== docker ps (caddy) ===" + docker ps -a --filter name=caddy echo "=== mnemonik-mcp-1 direct /health (docker exec localhost:3000) ===" docker exec mnemonik-mcp-1 curl -fsS -m 5 http://localhost:3000/health 2>&1; echo "(exec rc=$?)" echo "=== mnemonik-mcp-1 logs (tail 120) ===" docker logs mnemonik-mcp-1 --tail 120 2>&1 || true + echo "=== vaultwarden-caddy-1 logs (tail 120) ===" + docker logs vaultwarden-caddy-1 --tail 120 2>&1 || true register: mnemonik_diag changed_when: false failed_when: false diff --git a/infrastructure/ansible/roles/mnemonik-server/templates/Caddyfile.snippet.j2 b/infrastructure/ansible/roles/mnemonik-server/templates/Caddyfile.snippet.j2 index e1f7ef2..294bb37 100644 --- a/infrastructure/ansible/roles/mnemonik-server/templates/Caddyfile.snippet.j2 +++ b/infrastructure/ansible/roles/mnemonik-server/templates/Caddyfile.snippet.j2 @@ -19,7 +19,9 @@ {% if _cors_origins | length > 0 %} # CORS for the browser-based webapp frontend. # Preflight OPTIONS must short-circuit before reverse_proxy. - @cors_origin header Origin{% for origin in _cors_origins %} "{{ origin }}"{% endfor %} + # Use an expression matcher so multiple origins can be ORed; Caddy's + # `header` matcher only accepts a single value. + @cors_origin expression `{% for origin in _cors_origins %}{% if not loop.first %} || {% endif %}{http.request.header.Origin} == "{{ origin }}"{% endfor %}` @cors_preflight method OPTIONS handle @cors_preflight { diff --git a/infrastructure/ansible/roles/mnemonik-server/tests/test_role_contract.py b/infrastructure/ansible/roles/mnemonik-server/tests/test_role_contract.py index a3e4ad1..d56979d 100644 --- a/infrastructure/ansible/roles/mnemonik-server/tests/test_role_contract.py +++ b/infrastructure/ansible/roles/mnemonik-server/tests/test_role_contract.py @@ -137,7 +137,8 @@ def test_cors_enabled_when_origin_set(): mnemonik_mcp_container_port=3000, mnemonik_mcp_cors_origin="https://mnemonik.xyz", ) - assert 'header Origin "https://mnemonik.xyz"' in rendered + assert '@cors_origin expression' in rendered + assert '{http.request.header.Origin} == "https://mnemonik.xyz"' in rendered assert 'Access-Control-Allow-Origin "{http.request.header.Origin}"' in rendered assert "Access-Control-Allow-Methods" in rendered assert "Access-Control-Allow-Headers" in rendered @@ -159,7 +160,10 @@ def test_cors_enabled_with_multiple_origins(): "https://mnemonik-webapp.pages.dev", ], ) - assert 'header Origin "https://www.mnemonik.xyz" "https://mnemonik-webapp.pages.dev"' in rendered + assert '@cors_origin expression' in rendered + assert '{http.request.header.Origin} == "https://www.mnemonik.xyz"' in rendered + assert '{http.request.header.Origin} == "https://mnemonik-webapp.pages.dev"' in rendered + assert " || " in rendered assert 'Access-Control-Allow-Origin "{http.request.header.Origin}"' in rendered assert "@cors_preflight" in rendered From 1fd840d84afd0c948d3b0f9c98ade2c836a8439e Mon Sep 17 00:00:00 2001 From: mnemonik-dev Date: Sun, 12 Jul 2026 12:14:13 +0300 Subject: [PATCH 17/19] fix(deploy): remove stale mnemonik-mcp Caddy snippet before vaultwarden A prior failed mnemonik-server deploy left a broken Caddy snippet in the shared conf.d. vaultwarden starts Caddy before mnemonik-server runs, so the invalid snippet caused Caddy to crash-loop and the whole deploy to fail before mnemonik-server could rewrite it. Delete the stale snippet as a pre-task; mnemonik-server will render the fixed one later. --- infrastructure/ansible/playbooks/deploy.yml | 31 +++++++++++++++------ 1 file changed, 23 insertions(+), 8 deletions(-) diff --git a/infrastructure/ansible/playbooks/deploy.yml b/infrastructure/ansible/playbooks/deploy.yml index 35784f3..7febab7 100644 --- a/infrastructure/ansible/playbooks/deploy.yml +++ b/infrastructure/ansible/playbooks/deploy.yml @@ -40,6 +40,19 @@ APT_LISTCHANGES_FRONTEND: none pre_tasks: + # Recovery: a prior failed mnemonik-server deploy may have left a broken + # Caddy snippet in the shared conf.d. The vaultwarden role starts the + # shared Caddy before mnemonik-server has a chance to rewrite the snippet, + # so an invalid snippet blocks the entire deploy. Remove any stale + # mnemonik-mcp.conf; mnemonik-server will render a validated one later. + - name: Remove stale mnemonik-mcp Caddy snippet if present + ansible.builtin.file: + path: /opt/vaultwarden/caddy_conf_d/mnemonik-mcp.conf + state: absent + become: true + failed_when: false + changed_when: false + - name: Display deployment info debug: msg: | @@ -266,10 +279,12 @@ molyanov_skills_path: "/opt/molyanov/skills" molyanov_pk_guard_enabled: true - # Phase 6: Attestation + MCP (DESCOPED 2026-05 — disabled by default). - # The role still runs every deploy but installs only no-op stub hooks when - # `mnemonic_mcp_enabled` is false. Flip to true once the local Mnemonic MCP - # server is production-ready (backlog: work/mnemonic-attestation-integration/). + # Phase 6: Mnemonik MCP client binary (per-spawn stdio) for + # content-publisher attestation. Enabled by default; the discovered binary + # path is exposed as fact `mnemonic_mcp_binary` for downstream roles. + # (The legacy daemon + hook design was retired 2026-06; this role now + # installs only the npm-shipped client binary and runs an MCP handshake + # smoke test.) - role: mnemonic-mcp tags: [mnemonic-mcp, attestation] vars: @@ -411,14 +426,14 @@ - name: Verify all services are running block: - name: Check systemd services - # mnemonic-mcp only added to the loop when the role is enabled - # (descoped 2026-05; backlog: work/mnemonic-attestation-integration/) + # mnemonic-mcp is a per-spawn client binary, not a daemon; no + # systemd unit to check here. workspace-manager + telegram-ai-agent + # are the long-running fabric services. systemd: name: "{{ item }}" state: started register: service_check - loop: "{{ ['workspace-manager', 'telegram-ai-agent'] - + (['mnemonic-mcp'] if (mnemonic_mcp_enabled | default(false) | bool) else []) }}" + loop: "{{ ['workspace-manager', 'telegram-ai-agent'] }}" failed_when: false - name: Report service status From 98e2f6f870e389bcf4bfe45114932ac24ee93b32 Mon Sep 17 00:00:00 2001 From: mnemonik-dev Date: Sun, 12 Jul 2026 12:27:49 +0300 Subject: [PATCH 18/19] fix(mnemonik-server): point chat LLM at ollama service --- .../ansible/roles/mnemonik-server/templates/mcp.env.j2 | 3 +++ .../roles/mnemonik-server/tests/test_role_contract.py | 10 ++++++++++ 2 files changed, 13 insertions(+) diff --git a/infrastructure/ansible/roles/mnemonik-server/templates/mcp.env.j2 b/infrastructure/ansible/roles/mnemonik-server/templates/mcp.env.j2 index 35195f2..31753ce 100644 --- a/infrastructure/ansible/roles/mnemonik-server/templates/mcp.env.j2 +++ b/infrastructure/ansible/roles/mnemonik-server/templates/mcp.env.j2 @@ -6,6 +6,9 @@ # Topology: co-located ollama on the internal network. OLLAMA_URL=http://ollama:11434 OLLAMA_MODEL={{ mnemonik_ollama_model }} +LLM_PROVIDER=ollama +LLM_API_URL=http://ollama:11434 +LLM_MODEL={{ mnemonik_ollama_model }} # Server runtime. STORAGE_MODE={{ mnemonik_mcp_storage_mode }} diff --git a/infrastructure/ansible/roles/mnemonik-server/tests/test_role_contract.py b/infrastructure/ansible/roles/mnemonik-server/tests/test_role_contract.py index d56979d..841f74e 100644 --- a/infrastructure/ansible/roles/mnemonik-server/tests/test_role_contract.py +++ b/infrastructure/ansible/roles/mnemonik-server/tests/test_role_contract.py @@ -92,6 +92,16 @@ def test_ollama_model_pulled_not_built(): assert "ollama pull" in tasks +def test_chat_llm_points_at_ollama_container(): + # /chat uses the universal LLM client, which reads LLM_API_URL rather than + # OLLAMA_URL. Both must point at the co-located Ollama service. + env = _text("templates/mcp.env.j2") + assert "OLLAMA_URL=http://ollama:11434" in env + assert "LLM_PROVIDER=ollama" in env + assert "LLM_API_URL=http://ollama:11434" in env + assert "LLM_MODEL={{ mnemonik_ollama_model }}" in env + + def test_fastembed_cache_dir_explicit(): # FASTEMBED_CACHE_DIR must be set and point to the bind-mounted /data # subdir so model downloads survive VM rebuilds. From 9401e3c2076297c74eb862172c2f87890196b3d0 Mon Sep 17 00:00:00 2001 From: mnemonik-dev Date: Sun, 12 Jul 2026 12:31:54 +0300 Subject: [PATCH 19/19] fix(mnemonik-server): enable hosted anchoring config --- .../inventory/group_vars/all/features.yml | 16 ++++++++++ .../roles/mnemonik-server/defaults/main.yml | 3 ++ .../roles/mnemonik-server/tasks/main.yml | 29 +++++++++++++++++++ .../mnemonik-server/templates/mcp.env.j2 | 7 +++++ .../tests/test_role_contract.py | 15 ++++++++++ 5 files changed, 70 insertions(+) diff --git a/infrastructure/ansible/inventory/group_vars/all/features.yml b/infrastructure/ansible/inventory/group_vars/all/features.yml index 0a4892a..98cbd84 100644 --- a/infrastructure/ansible/inventory/group_vars/all/features.yml +++ b/infrastructure/ansible/inventory/group_vars/all/features.yml @@ -13,6 +13,22 @@ mnemonik_server_enabled: true # fail-soft GHCR login is skipped/ignored and the pull is anonymous. mnemonik_mcp_image_tag: "sha-61fdb3e" +# Hosted MCP is the public participate-mode operator. Default writes should +# anchor to Irys/Arweave + Solana; local-only writes remain available via +# explicit mode=local. +mnemonik_mcp_storage_mode: "full" + +# Wallets that have signed hosted anchored uploads. Include both the documented +# pre-Hetzner server identity and the accidental post-migration identity so the +# chain-backed public counters recover all indexed history. +mnemonik_chain_stats_wallets: >- + DYVu4Bry3BzGVsR3Hj2iGVT5fNdWFoHw2zRxsdTmrG25, + EdFyDHqqWTSPKs7c89U36Uy8QAnX2nX2A7qkYKD2Td21 + +mnemonik_arweave_url: "https://gateway.irys.xyz" +mnemonik_chain_stats_gateway_url: "https://gateway.irys.xyz" +mnemonik_chain_stats_graphql_url: "https://arweave.net/graphql" + # Browser webapp origins. Caddy reflects the request Origin if it matches # one of these exact origins. mnemonik_mcp_cors_origin: diff --git a/infrastructure/ansible/roles/mnemonik-server/defaults/main.yml b/infrastructure/ansible/roles/mnemonik-server/defaults/main.yml index 4adc390..ef8f3ea 100644 --- a/infrastructure/ansible/roles/mnemonik-server/defaults/main.yml +++ b/infrastructure/ansible/roles/mnemonik-server/defaults/main.yml @@ -59,6 +59,9 @@ mnemonik_mcp_fastembed_cache_dir: "/data/model-cache" # Defaults are safe/public; override in sops/group_vars for real tracking. mnemonik_chain_stats_wallets: "" mnemonik_solana_rpc_url: "https://api.mainnet-beta.solana.com" +mnemonik_arweave_url: "https://gateway.irys.xyz" +mnemonik_chain_stats_graphql_url: "https://arweave.net/graphql" +mnemonik_chain_stats_gateway_url: "https://gateway.irys.xyz" # Origin advertised in OAuth metadata + challenge envelopes. MUST match the # public hostname external clients (Cursor/Claude.ai) connect to — RFC 8707 # origin check is enforced. No trailing slash. diff --git a/infrastructure/ansible/roles/mnemonik-server/tasks/main.yml b/infrastructure/ansible/roles/mnemonik-server/tasks/main.yml index 8b687e9..9b66ae8 100644 --- a/infrastructure/ansible/roles/mnemonik-server/tasks/main.yml +++ b/infrastructure/ansible/roles/mnemonik-server/tasks/main.yml @@ -111,6 +111,35 @@ - keypair - ollama + - name: Check for legacy MCP identity filename + become: true + ansible.builtin.stat: + path: "{{ mnemonik_state_dir }}/keypair/id.json" + register: mnemonik_legacy_identity + + - name: Check for current MCP identity filename + become: true + ansible.builtin.stat: + path: "{{ mnemonik_state_dir }}/keypair/identity.json" + register: mnemonik_current_identity + + - name: Migrate legacy MCP identity filename to identity.json + # The Dockerfile used to advertise MNEMONIC_KEYPAIR_PATH=/keypair/id.json, + # but the current binary reads MNEMONIC_CONFIG_DIR/identity.json through + # identity::ensure(). Preserve an existing funded signer if it is present + # on the persistent volume. + become: true + ansible.builtin.copy: + src: "{{ mnemonik_state_dir }}/keypair/id.json" + dest: "{{ mnemonik_state_dir }}/keypair/identity.json" + remote_src: true + owner: "10001" + group: "999" + mode: "0600" + when: + - mnemonik_legacy_identity.stat.exists + - not mnemonik_current_identity.stat.exists + - name: Own MCP state dirs by the image's runtime uid (mnemonic = 10001:999) # The mcp image runs as uid 10001 / gid 999. Bind-mounting host dirs owned # by op(1000) made /data unreadable to the container ("Permission denied" diff --git a/infrastructure/ansible/roles/mnemonik-server/templates/mcp.env.j2 b/infrastructure/ansible/roles/mnemonik-server/templates/mcp.env.j2 index 31753ce..4ffedfa 100644 --- a/infrastructure/ansible/roles/mnemonik-server/templates/mcp.env.j2 +++ b/infrastructure/ansible/roles/mnemonik-server/templates/mcp.env.j2 @@ -10,6 +10,10 @@ LLM_PROVIDER=ollama LLM_API_URL=http://ollama:11434 LLM_MODEL={{ mnemonik_ollama_model }} +# Identity: the server code reads MNEMONIC_CONFIG_DIR/identity.json via +# identity::ensure(); MNEMONIC_KEYPAIR_PATH is not read by the current binary. +MNEMONIC_CONFIG_DIR=/keypair + # Server runtime. STORAGE_MODE={{ mnemonik_mcp_storage_mode }} EMBED_PROVIDER={{ mnemonik_mcp_embed_provider }} @@ -35,3 +39,6 @@ MCP_REFRESH_SALT={{ mnemonik_mcp_refresh_salt }} # Safe defaults point to a public RPC and an empty watch wallet list. CHAIN_STATS_WALLETS={{ mnemonik_chain_stats_wallets | default('') }} SOLANA_RPC_URL={{ mnemonik_solana_rpc_url | default('https://api.mainnet-beta.solana.com') }} +ARWEAVE_URL={{ mnemonik_arweave_url | default('https://gateway.irys.xyz') }} +CHAIN_STATS_GRAPHQL_URL={{ mnemonik_chain_stats_graphql_url | default('https://arweave.net/graphql') }} +CHAIN_STATS_GATEWAY_URL={{ mnemonik_chain_stats_gateway_url | default('https://gateway.irys.xyz') }} diff --git a/infrastructure/ansible/roles/mnemonik-server/tests/test_role_contract.py b/infrastructure/ansible/roles/mnemonik-server/tests/test_role_contract.py index 841f74e..a2d6cc1 100644 --- a/infrastructure/ansible/roles/mnemonik-server/tests/test_role_contract.py +++ b/infrastructure/ansible/roles/mnemonik-server/tests/test_role_contract.py @@ -75,6 +75,16 @@ def test_state_on_persistent_volume(): assert "/dev/sdb" in _text("tasks/main.yml") # discovers the volume mount +def test_identity_config_dir_is_persisted_keypair_mount(): + # The current binary ignores MNEMONIC_KEYPAIR_PATH and reads + # MNEMONIC_CONFIG_DIR/identity.json through identity::ensure(). + env = _text("templates/mcp.env.j2") + tasks = _text("tasks/main.yml") + assert "MNEMONIC_CONFIG_DIR=/keypair" in env + assert "keypair/id.json" in tasks + assert "keypair/identity.json" in tasks + + def test_secrets_asserted_and_have_no_default(): # Fails loudly if enabled without the hosted-mode secrets. defaults = _load("defaults/main.yml") @@ -116,8 +126,13 @@ def test_chain_stats_env_defaults(): env = _text("templates/mcp.env.j2") assert "CHAIN_STATS_WALLETS={{ mnemonik_chain_stats_wallets" in env assert "SOLANA_RPC_URL={{ mnemonik_solana_rpc_url" in env + assert "ARWEAVE_URL={{ mnemonik_arweave_url" in env + assert "CHAIN_STATS_GRAPHQL_URL={{ mnemonik_chain_stats_graphql_url" in env + assert "CHAIN_STATS_GATEWAY_URL={{ mnemonik_chain_stats_gateway_url" in env defaults = _load("defaults/main.yml") assert defaults["mnemonik_solana_rpc_url"] == "https://api.mainnet-beta.solana.com" + assert defaults["mnemonik_arweave_url"] == "https://gateway.irys.xyz" + assert defaults["mnemonik_chain_stats_gateway_url"] == "https://gateway.irys.xyz" def test_cors_disabled_by_default():