From d73520252c1816f62f910aad1e08243b3ebe18b6 Mon Sep 17 00:00:00 2001 From: Michael Johnson Date: Wed, 15 Jul 2026 19:36:29 +0100 Subject: [PATCH 01/10] Add Garage-backed artifact store and dual-target publish helper Store OATS run directories durably and serve them read-only over HTTP so the comparison viewer can enumerate and load runs. storage/ adds: - A single-node Garage (S3-compatible) service: compose.yaml, garage.toml (replication_factor = 1), and init.sh for the one-time layout / bucket / website / writer-key setup. Secrets are injected from a git-ignored .env; .env.example documents them. - publish_run.sh: idempotent, dual-target. Publishes a run directory to a filesystem root (rsync) or an S3 bucket (mc against Garage), keyed by the run's identity path, and regenerates a top-level index.json listing every run with its identity and aggregate pass/fail counts. The viewer reads only index.json plus the run objects; it never lists the bucket. - cors.xml and the website/anonymous-read setup give cross-origin GET for index.json, run_manifest.json, metrics and preview images. - README.md records the backend decision, local setup, publish workflow, credential handling and the runner deployment steps. Verified end-to-end locally against Garage: both publish targets, idempotent re-publish, and cross-origin browser fetch of the store over the web endpoint. --- README.md | 8 ++ storage/.env.example | 11 +++ storage/.gitignore | 2 + storage/README.md | 178 +++++++++++++++++++++++++++++++++++++++++ storage/compose.yaml | 18 +++++ storage/cors.xml | 13 +++ storage/garage.toml | 24 ++++++ storage/init.sh | 70 ++++++++++++++++ storage/publish_run.sh | 112 ++++++++++++++++++++++++++ 9 files changed, 436 insertions(+) create mode 100644 storage/.env.example create mode 100644 storage/.gitignore create mode 100644 storage/README.md create mode 100644 storage/compose.yaml create mode 100644 storage/cors.xml create mode 100644 storage/garage.toml create mode 100755 storage/init.sh create mode 100755 storage/publish_run.sh diff --git a/README.md b/README.md index ae73709..6cf9c41 100644 --- a/README.md +++ b/README.md @@ -118,6 +118,14 @@ If you want to aggregate all files into a single directory for ease of view, you This command will copy all odm_orthophoto.tif files from all test cases into `/my/path`. See `./harvest --help` for more options. +### Publishing runs to durable storage + +`results/runs/` is local and prunable. To retain runs and serve them read-only +over HTTP for the comparison viewer, publish them to the artifact store — a +Garage (S3-compatible) bucket or a plain filesystem root — with +`storage/publish_run.sh`. See [`storage/README.md`](storage/README.md) for the +backend, setup, and publish workflow. + ## Roadmap We have great plans for OATS. Some of them include: diff --git a/storage/.env.example b/storage/.env.example new file mode 100644 index 0000000..b6e7312 --- /dev/null +++ b/storage/.env.example @@ -0,0 +1,11 @@ +# Copy to storage/.env and fill in. storage/.env is git-ignored; never commit it. +# +# Generate fresh secrets with: +# GARAGE_RPC_SECRET -> openssl rand -hex 32 +# GARAGE_ADMIN_TOKEN -> openssl rand -base64 32 + +# Shared secret authenticating RPC between Garage nodes (required even single-node). +GARAGE_RPC_SECRET= + +# Bearer token for the Garage admin API on :3903. +GARAGE_ADMIN_TOKEN= diff --git a/storage/.gitignore b/storage/.gitignore new file mode 100644 index 0000000..a326095 --- /dev/null +++ b/storage/.gitignore @@ -0,0 +1,2 @@ +.env +compose.override.yaml diff --git a/storage/README.md b/storage/README.md new file mode 100644 index 0000000..313342f --- /dev/null +++ b/storage/README.md @@ -0,0 +1,178 @@ +# OATS run artifact storage + +Durable, addressable storage for OATS run directories, served read-only over +HTTP(S) so the comparison viewer can enumerate and load runs. + +**Backend: [Garage](https://garagehq.deuxfleurs.fr/)** — an S3-compatible object +store that ships as a single Rust binary (`dxflrs/garage`), run single-node with +`replication_factor = 1`. Chosen over the archived MinIO CE and a plain +nginx-over-a-directory: the S3 API lets the backend be swapped or grown +(multi-node / offsite) with only an endpoint change. CI and the viewer depend +only on the contract below, never on Garage itself. + +## Store contract + +The store mirrors the harness layout — no re-keying at publish time. Each run +sits at its identity path with a top-level `index.json` next to it: + +``` +oats-runs (bucket root) + index.json # every run, newest first + /// # one self-contained run + run_manifest.json + reports/*.xml + ///... # the ODM outputs +``` + +The key is the run directory's own relative path under `results/runs/` — read +straight from `run_manifest.json`'s `run_key`. + +`index.json` is the only file the viewer lists; it never enumerates the bucket: + +```json +{ "generated": "...", "runs": [ { + "key": "...", "timestamp": "...", "image": "...", "image_digest": "...", + "odm_git_revision": "...", "group": "...", + "datasets_total": 2, "passed": 0, "failed": 2 } ] } +``` + +`passed`/`failed` aggregate the per-dataset bats exit codes. + +## Local setup (development) + +Needs Docker. Secrets come from `storage/.env` (git-ignored — never commit). + +```sh +cd storage +cp .env.example .env # fill GARAGE_RPC_SECRET, GARAGE_ADMIN_TOKEN +docker compose up -d +./init.sh # idempotent: layout, bucket, website, writer key + # prints the writer key/secret once — save them +``` + +Then apply CORS so browsers can read cross-origin (needs an S3 client, e.g. +[`mc`](https://min.io/docs/minio/linux/reference/minio-mc.html)): + +```sh +mc alias set oats http://127.0.0.1:3900 +mc cors set oats/oats-runs cors.xml +``` + +Anonymous read is served by Garage's web endpoint (`127.0.0.1:3902`), which +resolves the bucket by virtual host: it matches the request `Host` (port +stripped) against a bucket global alias. `init.sh` creates the alias `oats-runs`; +add one per public hostname the store is served under (see deployment). To reach +it from a local browser, point that hostname at `127.0.0.1` (a `/etc/hosts` +entry) or go through the reverse proxy. + +## Publishing — `publish_run.sh` + +Publishes a run directory to the Garage bucket and regenerates `index.json`. +Idempotent per run key (`mc mirror --remove`); needs `mc` and `jq`. + +```sh +export AWS_ACCESS_KEY_ID= AWS_SECRET_ACCESS_KEY= +./publish_run.sh ../results/runs/// s3://oats-runs \ + --endpoint http://127.0.0.1:3900 +``` + +`mc` is found on `PATH` or via `OATS_MC=/path/to/mc`. + +## Credentials + +- `GARAGE_RPC_SECRET` / `GARAGE_ADMIN_TOKEN`: `storage/.env`, read by `compose.yaml`. +- Writer key/secret (from `init.sh`): the runner's CI secret store as + `AWS_ACCESS_KEY_ID` / `AWS_SECRET_ACCESS_KEY`, plus the endpoint as a variable. +- The viewer needs no credentials. + +Nothing secret is committed; `.env` and `compose.override.yaml` are git-ignored. + +## Deployment on the Proxmox server + +One always-on VM on the Proxmox host runs the CI job, Garage, and the viewer's +webserver. Garage stays bound to `127.0.0.1` (see `garage.toml`); the VM's +reverse proxy — the same one serving the viewer — fronts it publicly. VM +creation and GPU passthrough are the runner's concern, not the store's. + +### 1. Proxmox host — attach the data disk, start on boot + +Pass the spare drive through to the VM and make the VM come up after a host +reboot (so the store is always available): + +```sh +qm set -scsi1 /dev/disk/by-id/ # raw disk passthrough +qm set -onboot 1 +``` + +Snapshot before Garage upgrades for an instant rollback: +`qm snapshot pre-upgrade`. + +### 2. In the VM — mount the disk + +The passed-through drive shows up as a new block device (check `lsblk` — likely +`/dev/sdb`). Format it once and mount it where Garage's data will live: + +```sh +lsblk # confirm the device first! +sudo mkfs.ext4 /dev/sdb +sudo mkdir -p /data +echo '/dev/sdb /data ext4 defaults,nofail 0 2' | sudo tee -a /etc/fstab +sudo mount -a +``` + +### 3. In the VM — deploy Garage + +```sh +cd storage +printf 'GARAGE_RPC_SECRET=%s\nGARAGE_ADMIN_TOKEN=%s\n' \ + "$(openssl rand -hex 32)" "$(openssl rand -base64 32)" > .env + +# Put Garage's data on the mounted disk (git-ignored override): +cat > compose.override.yaml < +./mc cors set oats/oats-runs cors.xml +``` + +### 4. Front it with the VM's reverse proxy + +The web endpoint is on `127.0.0.1:3902`. Route the store's public hostname to it +through the same proxy that serves the viewer. Garage matches the request `Host` +against a bucket global alias, so alias the bucket to that hostname: + +```sh +docker compose exec garage /garage bucket alias oats-runs oats-store.example.com +``` + +Example nginx server block (store on its own subdomain): + +```nginx +server { + server_name oats-store.example.com; + location / { + proxy_pass http://127.0.0.1:3902; + proxy_set_header Host oats-store.example.com; + } +} +``` + +If the viewer instead serves the store under its own origin (same-origin path), +CORS isn't needed; on a separate hostname the committed `cors.xml` covers it. +TLS is terminated by the proxy (or by however the VM is exposed publicly), +alongside the viewer. + +### Durability + +Single node, single disk. Runs are re-derivable (re-run CI), or sync the bucket +offsite with `rclone sync`. Back up the guest's config with `vzdump`; don't +bother capturing the bulk data disk. diff --git a/storage/compose.yaml b/storage/compose.yaml new file mode 100644 index 0000000..9ef6456 --- /dev/null +++ b/storage/compose.yaml @@ -0,0 +1,18 @@ +services: + garage: + image: dxflrs/garage:v1.0.1 + container_name: oats-garage + restart: unless-stopped + network_mode: host + environment: + # Secrets are injected from storage/.env (never committed). See README. + GARAGE_RPC_SECRET: ${GARAGE_RPC_SECRET:?set GARAGE_RPC_SECRET in storage/.env} + GARAGE_ADMIN_TOKEN: ${GARAGE_ADMIN_TOKEN:?set GARAGE_ADMIN_TOKEN in storage/.env} + volumes: + - ./garage.toml:/etc/garage.toml:ro + - meta:/var/lib/garage/meta + - data:/var/lib/garage/data + +volumes: + meta: + data: diff --git a/storage/cors.xml b/storage/cors.xml new file mode 100644 index 0000000..221da39 --- /dev/null +++ b/storage/cors.xml @@ -0,0 +1,13 @@ + + + + * + GET + HEAD + * + ETag + Content-Length + Content-Type + 3600 + + diff --git a/storage/garage.toml b/storage/garage.toml new file mode 100644 index 0000000..8910c73 --- /dev/null +++ b/storage/garage.toml @@ -0,0 +1,24 @@ +metadata_dir = "/var/lib/garage/meta" +data_dir = "/var/lib/garage/data" +db_engine = "lmdb" + +replication_factor = 1 + +# Everything binds to localhost. Public read is served only through the host's +# reverse proxy (see README), which reaches the web endpoint over 127.0.0.1 — so +# nothing is exposed on the network directly. +rpc_bind_addr = "127.0.0.1:3901" +rpc_public_addr = "127.0.0.1:3901" + +[s3_api] +s3_region = "garage" +api_bind_addr = "127.0.0.1:3900" +root_domain = ".s3.garage.localhost" + +[s3_web] +bind_addr = "127.0.0.1:3902" +root_domain = ".web.garage.localhost" +index = "index.html" + +[admin] +api_bind_addr = "127.0.0.1:3903" diff --git a/storage/init.sh b/storage/init.sh new file mode 100755 index 0000000..982121c --- /dev/null +++ b/storage/init.sh @@ -0,0 +1,70 @@ +#!/bin/bash +# +# One-time initialisation of the Garage cluster used to store OATS runs. +# Idempotent: safe to re-run. Performs the Garage-CLI setup (single-node +# layout, bucket, website hosting, writer key). The CORS step needs an S3 +# client and is documented separately in README.md. +# +# Run from the storage/ directory after `docker compose up -d`: +# ./init.sh +# +# Prints the writer key credentials at the end. Store them in an env file / +# CI secret (see README) -- never commit them. + +set -euo pipefail + +BUCKET="${OATS_BUCKET:-oats-runs}" +KEY_NAME="${OATS_KEY_NAME:-oats-writer}" +CAPACITY="${OATS_CAPACITY:-100G}" +ZONE="${OATS_ZONE:-dc1}" + +g() { docker compose exec -T -e RUST_LOG=error garage /garage "$@"; } + +echo "Waiting for Garage to be reachable..." +for _ in $(seq 1 30); do + if g status >/dev/null 2>&1; then break; fi + sleep 1 +done + +# Assign a layout to the single node if it has no role yet. +node=$(g status 2>/dev/null | awk '/NO ROLE ASSIGNED/{print $1}') +if [ -n "$node" ]; then + echo "Assigning layout to node $node ($CAPACITY, zone $ZONE)..." + g layout assign -z "$ZONE" -c "$CAPACITY" "$node" >/dev/null + ver=$(g layout show 2>/dev/null | awk '/Current cluster layout version/{print $NF}') + g layout apply --version "$((ver + 1))" >/dev/null + echo "Layout applied." +else + echo "Node already has a role; skipping layout assignment." +fi + +# Create the bucket and enable public website (anonymous read) hosting. +if ! g bucket info "$BUCKET" >/dev/null 2>&1; then + echo "Creating bucket $BUCKET..." + g bucket create "$BUCKET" >/dev/null +fi +g bucket website --allow "$BUCKET" >/dev/null +echo "Bucket $BUCKET ready with website (anonymous read) enabled." + +# Create the writer key if it does not exist and grant it access. +if g key info "$KEY_NAME" >/dev/null 2>&1; then + echo "Key $KEY_NAME already exists; not recreating (its secret is only shown at creation)." + g bucket allow --read --write --owner "$BUCKET" --key "$KEY_NAME" >/dev/null +else + echo "Creating writer key $KEY_NAME..." + created=$(g key create "$KEY_NAME" 2>/dev/null) + g bucket allow --read --write --owner "$BUCKET" --key "$KEY_NAME" >/dev/null + kid=$(echo "$created" | awk -F': ' '/Key ID/{print $2}') + ksec=$(echo "$created" | awk -F': ' '/Secret key/{print $2}') + echo + echo "==========================================================" + echo "Writer key created -- store these, they are shown ONCE:" + echo " AWS_ACCESS_KEY_ID=$kid" + echo " AWS_SECRET_ACCESS_KEY=$ksec" + echo "==========================================================" +fi + +echo +echo "Next: apply CORS so browsers can read runs (see README):" +echo " mc alias set oats http://localhost:3900 " +echo " mc cors set oats/$BUCKET cors.xml" diff --git a/storage/publish_run.sh b/storage/publish_run.sh new file mode 100755 index 0000000..4531702 --- /dev/null +++ b/storage/publish_run.sh @@ -0,0 +1,112 @@ +#!/bin/bash +# +# Publish an OATS run directory to the Garage (S3) store and regenerate the +# store's index.json, using mc (MinIO client) against the S3 endpoint. +# +# The store mirrors the harness layout: a run keyed by its identity +# (//) lives at the bucket root next to a +# top-level index.json listing every run. Re-publishing the same run overwrites +# it cleanly, so the operation is idempotent per run key. +# +# Usage: +# publish_run.sh --endpoint URL +# +# A run directory containing run_manifest.json, e.g. +# results/runs/// +# Bucket URL, e.g. s3://oats-runs +# +# Options: +# --endpoint URL S3 endpoint, e.g. http://127.0.0.1:3900 (required) +# --mc PATH mc binary to use (default: mc on PATH, or $OATS_MC) +# -h, --help Show this help +# +# S3 credentials are read from the environment: +# AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY + +set -euo pipefail + +usage() { + sed -n '2,/^set -euo/p' "$0" | sed 's/^#\?//;s/^ //' | head -n -1 + exit "${1:-0}" +} + +RUN_DIR="" +TARGET="" +ENDPOINT="" +MC="${OATS_MC:-mc}" + +while [ $# -gt 0 ]; do + case "$1" in + --endpoint) ENDPOINT="$2"; shift 2;; + --mc) MC="$2"; shift 2;; + -h|--help) usage 0;; + -*) echo "Unknown option: $1" >&2; usage 1;; + *) + if [ -z "$RUN_DIR" ]; then RUN_DIR="$1" + elif [ -z "$TARGET" ]; then TARGET="$1" + else echo "Unexpected argument: $1" >&2; usage 1 + fi + shift;; + esac +done + +[ -n "$RUN_DIR" ] && [ -n "$TARGET" ] || usage 1 +case "$TARGET" in + s3://*) ;; + *) echo "Target must be an s3:// URL (e.g. s3://oats-runs)" >&2; usage 1;; +esac +[ -n "$ENDPOINT" ] || { echo "--endpoint is required" >&2; usage 1; } +[ -n "${AWS_ACCESS_KEY_ID:-}" ] && [ -n "${AWS_SECRET_ACCESS_KEY:-}" ] \ + || { echo "AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY must be set" >&2; exit 1; } + +command -v jq >/dev/null || { echo "jq is required" >&2; exit 1; } +command -v "$MC" >/dev/null || { echo "mc client not found: $MC" >&2; exit 1; } + +MANIFEST="$RUN_DIR/run_manifest.json" +[ -f "$MANIFEST" ] || { echo "No run_manifest.json in $RUN_DIR" >&2; exit 1; } + +# The run key is the run's identity path, written by the harness. Fall back to +# the run directory's own trailing // path if absent. +RUN_KEY=$(jq -r '.run_key // empty' "$MANIFEST") +if [ -z "$RUN_KEY" ]; then + abs=$(cd "$RUN_DIR" && pwd) + RUN_KEY=$(echo "$abs" | rev | cut -d/ -f1-3 | rev) +fi +[ -n "$RUN_KEY" ] || { echo "Could not determine run key" >&2; exit 1; } + +# jq program turning a stream of run manifests into the store index. Each run +# contributes its identity plus an aggregate pass/fail count derived from the +# per-dataset bats exit codes. +INDEX_JQ='{ + generated: (now | todateiso8601), + runs: [ .[] | { + key: .run_key, + timestamp: .timestamp, + image: .image, + image_digest: .image_digest, + odm_git_revision: .odm_git_revision, + group: .group, + datasets_total: (.datasets | length), + passed: ([.datasets[] | select(.bats_exit == 0)] | length), + failed: ([.datasets[] | select(.bats_exit != 0)] | length) + } ] | sort_by(.timestamp) | reverse +}' + +# bucket[/prefix] from s3://bucket[/prefix] +path="${TARGET#s3://}" +alias="oats-publish-$$" +base="$alias/$path" + +cfg=$(mktemp -d) +trap 'rm -rf "$cfg"' EXIT +"$MC" --config-dir "$cfg" alias set "$alias" "$ENDPOINT" \ + "$AWS_ACCESS_KEY_ID" "$AWS_SECRET_ACCESS_KEY" >/dev/null + +"$MC" --config-dir "$cfg" mirror --overwrite --remove "$RUN_DIR/" "$base/$RUN_KEY" >/dev/null +"$MC" --config-dir "$cfg" find "$base" --name run_manifest.json 2>/dev/null \ + | while IFS= read -r p; do "$MC" --config-dir "$cfg" cat "$p"; done \ + | jq -s "$INDEX_JQ" \ + | "$MC" --config-dir "$cfg" pipe "$base/index.json" >/dev/null + +echo "Published run $RUN_KEY to $TARGET/$RUN_KEY" +echo "Regenerated $TARGET/index.json" From e8396a779294977ddf62c77d07871fc5c596c507 Mon Sep 17 00:00:00 2001 From: Michael Johnson Date: Wed, 19 Aug 2026 14:22:01 +0100 Subject: [PATCH 02/10] Deploy the artifact store with Ansible instead of a runbook The store's deployment was a 89-line README procedure: pass the disk through, mkfs and fstab it, generate secrets, write a compose override, install mc, apply CORS, alias the bucket, add an nginx vhost. Every step was a copy-paste command with no way to re-run it safely. deploy/ replaces that with an idempotent Ansible role covering the whole host setup, so rolling out a config change is one command. init.sh is reused rather than reimplemented, keeping one definition of the cluster setup for both local development and deployment. Writer credentials are generated on first run and kept root-only in /etc/oats/writer.env, with an opt-in fetch for pasting into the CI secret store. Only the two Proxmox host operations stay manual, being one-shot and outside the VM. The README drops to the store contract, the deployment command and the parts that need explaining rather than executing. Verified against a local Garage: fresh deploy, a re-run reporting no changes, a run published through the deployed store, and a cross-origin read of index.json over the aliased public hostname. --- storage/.gitignore | 2 + storage/README.md | 158 ++++---------- storage/deploy/ansible.cfg | 4 + storage/deploy/inventory.example.yml | 21 ++ storage/deploy/requirements.yml | 8 + .../roles/oats_storage/defaults/main.yml | 37 ++++ .../roles/oats_storage/handlers/main.yml | 10 + .../deploy/roles/oats_storage/tasks/main.yml | 201 ++++++++++++++++++ .../templates/compose.override.yaml.j2 | 6 + .../templates/nginx-store.conf.j2 | 14 ++ storage/deploy/storage.yml | 10 + 11 files changed, 360 insertions(+), 111 deletions(-) create mode 100644 storage/deploy/ansible.cfg create mode 100644 storage/deploy/inventory.example.yml create mode 100644 storage/deploy/requirements.yml create mode 100644 storage/deploy/roles/oats_storage/defaults/main.yml create mode 100644 storage/deploy/roles/oats_storage/handlers/main.yml create mode 100644 storage/deploy/roles/oats_storage/tasks/main.yml create mode 100644 storage/deploy/roles/oats_storage/templates/compose.override.yaml.j2 create mode 100644 storage/deploy/roles/oats_storage/templates/nginx-store.conf.j2 create mode 100644 storage/deploy/storage.yml diff --git a/storage/.gitignore b/storage/.gitignore index a326095..ea2e739 100644 --- a/storage/.gitignore +++ b/storage/.gitignore @@ -1,2 +1,4 @@ .env compose.override.yaml +deploy/inventory.yml +deploy/writer.env diff --git a/storage/README.md b/storage/README.md index 313342f..8561501 100644 --- a/storage/README.md +++ b/storage/README.md @@ -27,7 +27,8 @@ oats-runs (bucket root) The key is the run directory's own relative path under `results/runs/` — read straight from `run_manifest.json`'s `run_key`. -`index.json` is the only file the viewer lists; it never enumerates the bucket: +`index.json` is the only file the viewer reads to discover runs; it never +enumerates the bucket: ```json { "generated": "...", "runs": [ { @@ -38,140 +39,75 @@ straight from `run_manifest.json`'s `run_key`. `passed`/`failed` aggregate the per-dataset bats exit codes. -## Local setup (development) +## Deployment -Needs Docker. Secrets come from `storage/.env` (git-ignored — never commit). +`deploy/` holds an Ansible role that does the whole host setup: mounts the data +disk, deploys this directory, generates Garage's secrets, starts the container, +runs `init.sh`, applies CORS, and aliases the bucket to the public hostname +(optionally writing the nginx vhost). It is idempotent — re-run it to roll out a +config change. ```sh -cd storage -cp .env.example .env # fill GARAGE_RPC_SECRET, GARAGE_ADMIN_TOKEN -docker compose up -d -./init.sh # idempotent: layout, bucket, website, writer key - # prints the writer key/secret once — save them +cd deploy +ansible-galaxy collection install -r requirements.yml +cp inventory.example.yml inventory.yml # set the host, disk and hostname +ansible-playbook storage.yml ``` -Then apply CORS so browsers can read cross-origin (needs an S3 client, e.g. -[`mc`](https://min.io/docs/minio/linux/reference/minio-mc.html)): - -```sh -mc alias set oats http://127.0.0.1:3900 -mc cors set oats/oats-runs cors.xml -``` - -Anonymous read is served by Garage's web endpoint (`127.0.0.1:3902`), which -resolves the bucket by virtual host: it matches the request `Host` (port -stripped) against a bucket global alias. `init.sh` creates the alias `oats-runs`; -add one per public hostname the store is served under (see deployment). To reach -it from a local browser, point that hostname at `127.0.0.1` (a `/etc/hosts` -entry) or go through the reverse proxy. - -## Publishing — `publish_run.sh` +The writer credentials are generated on first run and left in +`/etc/oats/writer.env` on the host (root-only); pass `-e oats_fetch_writer_env=true` +to copy them back for pasting into the CI secret store. Garage only reveals a +secret at creation, so if that file is lost the key has to be reissued. -Publishes a run directory to the Garage bucket and regenerates `index.json`. -Idempotent per run key (`mc mirror --remove`); needs `mc` and `jq`. +Two things stay manual, on the Proxmox host — they are one-shot and outside the +VM: ```sh -export AWS_ACCESS_KEY_ID= AWS_SECRET_ACCESS_KEY= -./publish_run.sh ../results/runs/// s3://oats-runs \ - --endpoint http://127.0.0.1:3900 +qm set -scsi1 /dev/disk/by-id/ # pass the data disk through +qm set -onboot 1 # store is always-on ``` -`mc` is found on `PATH` or via `OATS_MC=/path/to/mc`. +Snapshot before a Garage upgrade (`qm snapshot pre-upgrade`) for instant +rollback. -## Credentials - -- `GARAGE_RPC_SECRET` / `GARAGE_ADMIN_TOKEN`: `storage/.env`, read by `compose.yaml`. -- Writer key/secret (from `init.sh`): the runner's CI secret store as - `AWS_ACCESS_KEY_ID` / `AWS_SECRET_ACCESS_KEY`, plus the endpoint as a variable. -- The viewer needs no credentials. - -Nothing secret is committed; `.env` and `compose.override.yaml` are git-ignored. - -## Deployment on the Proxmox server +> Status: the role is verified end-to-end against a local Garage (fresh deploy, +> idempotent re-run, publish, cross-origin read). Running it against the +> Proxmox VM is the one remaining step, pending access to that host. -One always-on VM on the Proxmox host runs the CI job, Garage, and the viewer's -webserver. Garage stays bound to `127.0.0.1` (see `garage.toml`); the VM's -reverse proxy — the same one serving the viewer — fronts it publicly. VM -creation and GPU passthrough are the runner's concern, not the store's. +## Local development -### 1. Proxmox host — attach the data disk, start on boot - -Pass the spare drive through to the VM and make the VM come up after a host -reboot (so the store is always available): +Docker only — no Ansible needed: ```sh -qm set -scsi1 /dev/disk/by-id/ # raw disk passthrough -qm set -onboot 1 -``` - -Snapshot before Garage upgrades for an instant rollback: -`qm snapshot pre-upgrade`. - -### 2. In the VM — mount the disk - -The passed-through drive shows up as a new block device (check `lsblk` — likely -`/dev/sdb`). Format it once and mount it where Garage's data will live: - -```sh -lsblk # confirm the device first! -sudo mkfs.ext4 /dev/sdb -sudo mkdir -p /data -echo '/dev/sdb /data ext4 defaults,nofail 0 2' | sudo tee -a /etc/fstab -sudo mount -a -``` - -### 3. In the VM — deploy Garage - -```sh -cd storage -printf 'GARAGE_RPC_SECRET=%s\nGARAGE_ADMIN_TOKEN=%s\n' \ - "$(openssl rand -hex 32)" "$(openssl rand -base64 32)" > .env - -# Put Garage's data on the mounted disk (git-ignored override): -cat > compose.override.yaml < -./mc cors set oats/oats-runs cors.xml +./init.sh # idempotent; prints the writer key once +mc alias set oats http://127.0.0.1:3900 +mc cors set oats/oats-runs cors.xml ``` -### 4. Front it with the VM's reverse proxy +Garage resolves a bucket from the request `Host`, matched against a bucket +global alias. `init.sh` creates the alias `oats-runs`; to reach the web endpoint +(`127.0.0.1:3902`) from a browser, point that name at `127.0.0.1` in +`/etc/hosts`, or add an alias for whatever hostname you use. -The web endpoint is on `127.0.0.1:3902`. Route the store's public hostname to it -through the same proxy that serves the viewer. Garage matches the request `Host` -against a bucket global alias, so alias the bucket to that hostname: +## Publishing -```sh -docker compose exec garage /garage bucket alias oats-runs oats-store.example.com -``` +`./publish_run.sh s3://oats-runs --endpoint http://127.0.0.1:3900` +writes a run and regenerates `index.json`. Idempotent per run key; needs `mc` +and `jq`. Credentials come from `AWS_ACCESS_KEY_ID` / `AWS_SECRET_ACCESS_KEY`. +See `--help`. -Example nginx server block (store on its own subdomain): +## Credentials -```nginx -server { - server_name oats-store.example.com; - location / { - proxy_pass http://127.0.0.1:3902; - proxy_set_header Host oats-store.example.com; - } -} -``` +- `GARAGE_RPC_SECRET` / `GARAGE_ADMIN_TOKEN`: `storage/.env`, read by `compose.yaml`. +- Writer key/secret: `/etc/oats/writer.env` on the host, and the CI secret store. +- The viewer needs none — it reads the public endpoint only. -If the viewer instead serves the store under its own origin (same-origin path), -CORS isn't needed; on a separate hostname the committed `cors.xml` covers it. -TLS is terminated by the proxy (or by however the VM is exposed publicly), -alongside the viewer. +Nothing secret is committed; `.env`, `compose.override.yaml`, `deploy/inventory.yml` +and `deploy/writer.env` are git-ignored. -### Durability +## Durability Single node, single disk. Runs are re-derivable (re-run CI), or sync the bucket offsite with `rclone sync`. Back up the guest's config with `vzdump`; don't diff --git a/storage/deploy/ansible.cfg b/storage/deploy/ansible.cfg new file mode 100644 index 0000000..2977909 --- /dev/null +++ b/storage/deploy/ansible.cfg @@ -0,0 +1,4 @@ +[defaults] +inventory = inventory.yml +roles_path = roles +stdout_callback = yaml diff --git a/storage/deploy/inventory.example.yml b/storage/deploy/inventory.example.yml new file mode 100644 index 0000000..4c1595d --- /dev/null +++ b/storage/deploy/inventory.example.yml @@ -0,0 +1,21 @@ +--- +# Copy to inventory.yml (git-ignored) and edit. One host: the always-on VM that +# runs CI, Garage and the viewer. +all: + hosts: + oats-store: + ansible_host: oats-vm.lan + ansible_user: michael + + # Block device passed through from the Proxmox host. Leave unset to skip + # all disk handling and use whatever is already mounted at oats_data_root. + # WARNING: when set and unformatted, the playbook formats this device. + # oats_data_device: /dev/sdb + + # Public hostname the store is served under. When set, the bucket gains a + # matching alias so Garage resolves it by virtual host. + # oats_public_hostname: oats-store.example.com + + # Let the playbook write the nginx vhost too. Leave false if the viewer's + # task owns the proxy config. + # oats_nginx_manage: true diff --git a/storage/deploy/requirements.yml b/storage/deploy/requirements.yml new file mode 100644 index 0000000..e6d2e76 --- /dev/null +++ b/storage/deploy/requirements.yml @@ -0,0 +1,8 @@ +--- +collections: + - name: ansible.posix + version: ">=1.5.0" + - name: community.docker + version: ">=3.6.0" + - name: community.general + version: ">=8.0.0" diff --git a/storage/deploy/roles/oats_storage/defaults/main.yml b/storage/deploy/roles/oats_storage/defaults/main.yml new file mode 100644 index 0000000..4880ae8 --- /dev/null +++ b/storage/deploy/roles/oats_storage/defaults/main.yml @@ -0,0 +1,37 @@ +--- +# Where the committed store config (compose.yaml, garage.toml, cors.xml, +# init.sh) is deployed to on the host. +oats_storage_dir: /opt/oats-storage + +# Garage's metadata and data live here, on the data disk. +oats_data_mount: /data +oats_data_root: "{{ oats_data_mount }}/oats-garage" + +# Optional block device to format and mount at oats_data_mount. Unset means +# the playbook touches no disks. +oats_data_device: "" +oats_data_fstype: ext4 + +oats_bucket: oats-runs +oats_s3_endpoint: http://127.0.0.1:3900 +oats_web_endpoint: http://127.0.0.1:3902 + +# Credentials written here, root-only. The writer secret is shown once by +# init.sh, so this file is the only copy on the host. +oats_conf_dir: /etc/oats +oats_writer_env: "{{ oats_conf_dir }}/writer.env" + +# Copy the writer credentials back to the control node (for pasting into the +# CI secret store). Off by default so secrets do not spread silently. +oats_fetch_writer_env: false +oats_fetch_dest: ./writer.env + +oats_mc_path: /usr/local/bin/mc +oats_mc_url: https://dl.min.io/client/mc/release/linux-amd64/mc + +# Public hostname; when set the bucket is aliased to it so Garage resolves it +# by virtual host. nginx vhost is only written when oats_nginx_manage is true. +oats_public_hostname: "" +oats_nginx_manage: false +oats_nginx_site_dir: /etc/nginx/sites-available +oats_nginx_enabled_dir: /etc/nginx/sites-enabled diff --git a/storage/deploy/roles/oats_storage/handlers/main.yml b/storage/deploy/roles/oats_storage/handlers/main.yml new file mode 100644 index 0000000..1f05d17 --- /dev/null +++ b/storage/deploy/roles/oats_storage/handlers/main.yml @@ -0,0 +1,10 @@ +--- +- name: Restart Garage + community.docker.docker_compose_v2: + project_src: "{{ oats_storage_dir }}" + state: restarted + +- name: Reload nginx + ansible.builtin.systemd: + name: nginx + state: reloaded diff --git a/storage/deploy/roles/oats_storage/tasks/main.yml b/storage/deploy/roles/oats_storage/tasks/main.yml new file mode 100644 index 0000000..f30e43d --- /dev/null +++ b/storage/deploy/roles/oats_storage/tasks/main.yml @@ -0,0 +1,201 @@ +--- +- name: Check that Docker is installed + ansible.builtin.command: docker compose version + changed_when: false + register: oats_docker_check + failed_when: oats_docker_check.rc != 0 + +- name: Prepare the data disk + when: oats_data_device | length > 0 + block: + - name: Create a filesystem on the data device + community.general.filesystem: + fstype: "{{ oats_data_fstype }}" + dev: "{{ oats_data_device }}" + + - name: Mount the data device + ansible.posix.mount: + path: "{{ oats_data_mount }}" + src: "{{ oats_data_device }}" + fstype: "{{ oats_data_fstype }}" + opts: defaults,nofail + state: mounted + +- name: Create the store directories + ansible.builtin.file: + path: "{{ item.path }}" + state: directory + mode: "{{ item.mode }}" + loop: + - { path: "{{ oats_storage_dir }}", mode: "0755" } + - { path: "{{ oats_data_root }}/meta", mode: "0700" } + - { path: "{{ oats_data_root }}/data", mode: "0700" } + - { path: "{{ oats_conf_dir }}", mode: "0700" } + - { path: "{{ oats_conf_dir }}/mc", mode: "0700" } + +- name: Deploy the store configuration + ansible.builtin.copy: + src: "{{ playbook_dir }}/../{{ item.name }}" + dest: "{{ oats_storage_dir }}/{{ item.name }}" + mode: "{{ item.mode }}" + loop: + - { name: compose.yaml, mode: "0644" } + - { name: garage.toml, mode: "0644" } + - { name: cors.xml, mode: "0644" } + - { name: init.sh, mode: "0755" } + - { name: publish_run.sh, mode: "0755" } + notify: Restart Garage + +- name: Point Garage's volumes at the data disk + ansible.builtin.template: + src: compose.override.yaml.j2 + dest: "{{ oats_storage_dir }}/compose.override.yaml" + mode: "0644" + notify: Restart Garage + +# Garage's own secrets are generated once and never rotated by a re-run. +- name: Look for existing Garage secrets + ansible.builtin.stat: + path: "{{ oats_storage_dir }}/.env" + register: oats_env_file + +- name: Generate Garage secrets + ansible.builtin.set_fact: + oats_rpc_secret: "{{ lookup('ansible.builtin.password', '/dev/null', chars=['hexdigits'], length=64) | lower }}" + oats_admin_token: "{{ lookup('ansible.builtin.password', '/dev/null', length=32) }}" + when: not oats_env_file.stat.exists + no_log: true + +- name: Write the Garage secrets file + ansible.builtin.copy: + content: | + GARAGE_RPC_SECRET={{ oats_rpc_secret }} + GARAGE_ADMIN_TOKEN={{ oats_admin_token }} + dest: "{{ oats_storage_dir }}/.env" + mode: "0600" + when: not oats_env_file.stat.exists + no_log: true + +- name: Start Garage + community.docker.docker_compose_v2: + project_src: "{{ oats_storage_dir }}" + state: present + +- name: Flush handlers so Garage restarts before it is initialised + ansible.builtin.meta: flush_handlers + +# init.sh is idempotent and is the same script used for local development, so +# the layout / bucket / website / key setup has exactly one definition. +- name: Initialise the cluster, bucket and writer key + ansible.builtin.command: ./init.sh + args: + chdir: "{{ oats_storage_dir }}" + register: oats_init + changed_when: "'Writer key created' in oats_init.stdout" + no_log: true + +- name: Store the writer credentials + ansible.builtin.copy: + content: | + AWS_ACCESS_KEY_ID={{ oats_init.stdout | regex_search('AWS_ACCESS_KEY_ID=(\S+)', '\1') | first }} + AWS_SECRET_ACCESS_KEY={{ oats_init.stdout | regex_search('AWS_SECRET_ACCESS_KEY=(\S+)', '\1') | first }} + dest: "{{ oats_writer_env }}" + mode: "0600" + when: "'Writer key created' in oats_init.stdout" + no_log: true + +- name: Check that the writer credentials are on disk + ansible.builtin.stat: + path: "{{ oats_writer_env }}" + register: oats_writer_stat + +- name: Fail when the writer secret is unrecoverable + ansible.builtin.fail: + msg: >- + The writer key exists in Garage but {{ oats_writer_env }} is missing, and + Garage only reveals a secret at creation time. Delete the key + ("garage key delete oats-writer") and re-run to issue a new one. + when: not oats_writer_stat.stat.exists + +- name: Read the writer credentials + ansible.builtin.slurp: + src: "{{ oats_writer_env }}" + register: oats_writer_raw + no_log: true + +- name: Load the writer credentials + ansible.builtin.set_fact: + oats_key_id: "{{ oats_writer_raw.content | b64decode | regex_search('AWS_ACCESS_KEY_ID=(\\S+)', '\\1') | first }}" + oats_key_secret: "{{ oats_writer_raw.content | b64decode | regex_search('AWS_SECRET_ACCESS_KEY=(\\S+)', '\\1') | first }}" + no_log: true + +- name: Install the mc S3 client + ansible.builtin.get_url: + url: "{{ oats_mc_url }}" + dest: "{{ oats_mc_path }}" + mode: "0755" + +- name: Register the bucket with mc + ansible.builtin.command: >- + {{ oats_mc_path }} --config-dir {{ oats_conf_dir }}/mc alias set oats + {{ oats_s3_endpoint }} {{ oats_key_id }} {{ oats_key_secret }} + changed_when: false + no_log: true + +- name: Read the current CORS configuration + ansible.builtin.command: >- + {{ oats_mc_path }} --config-dir {{ oats_conf_dir }}/mc + cors get oats/{{ oats_bucket }} + register: oats_cors_now + changed_when: false + failed_when: false + +- name: Allow cross-origin reads + ansible.builtin.command: >- + {{ oats_mc_path }} --config-dir {{ oats_conf_dir }}/mc cors set oats/{{ oats_bucket }} + {{ oats_storage_dir }}/cors.xml + when: "'AllowedOrigin' not in oats_cors_now.stdout" + +- name: Serve the bucket under the public hostname + when: oats_public_hostname | length > 0 + block: + # Garage resolves a bucket from the request Host, so the public hostname + # has to exist as a bucket alias. + - name: Read the bucket's aliases + ansible.builtin.command: docker compose exec -T garage /garage bucket info {{ oats_bucket }} + args: + chdir: "{{ oats_storage_dir }}" + register: oats_bucket_info + changed_when: false + + - name: Alias the bucket to the public hostname + ansible.builtin.command: >- + docker compose exec -T garage /garage bucket alias + {{ oats_bucket }} {{ oats_public_hostname }} + args: + chdir: "{{ oats_storage_dir }}" + when: oats_public_hostname not in oats_bucket_info.stdout + + - name: Write the nginx vhost + when: oats_nginx_manage | bool + block: + - name: Install the store's server block + ansible.builtin.template: + src: nginx-store.conf.j2 + dest: "{{ oats_nginx_site_dir }}/oats-store" + mode: "0644" + notify: Reload nginx + + - name: Enable the store's server block + ansible.builtin.file: + src: "{{ oats_nginx_site_dir }}/oats-store" + dest: "{{ oats_nginx_enabled_dir }}/oats-store" + state: link + notify: Reload nginx + +- name: Copy the writer credentials to the control node + ansible.builtin.fetch: + src: "{{ oats_writer_env }}" + dest: "{{ oats_fetch_dest }}" + flat: true + when: oats_fetch_writer_env | bool diff --git a/storage/deploy/roles/oats_storage/templates/compose.override.yaml.j2 b/storage/deploy/roles/oats_storage/templates/compose.override.yaml.j2 new file mode 100644 index 0000000..153a591 --- /dev/null +++ b/storage/deploy/roles/oats_storage/templates/compose.override.yaml.j2 @@ -0,0 +1,6 @@ +# Managed by Ansible (roles/oats_storage). Local edits are overwritten. +services: + garage: + volumes: + - {{ oats_data_root }}/meta:/var/lib/garage/meta + - {{ oats_data_root }}/data:/var/lib/garage/data diff --git a/storage/deploy/roles/oats_storage/templates/nginx-store.conf.j2 b/storage/deploy/roles/oats_storage/templates/nginx-store.conf.j2 new file mode 100644 index 0000000..d962cfa --- /dev/null +++ b/storage/deploy/roles/oats_storage/templates/nginx-store.conf.j2 @@ -0,0 +1,14 @@ +# Managed by Ansible (roles/oats_storage). Local edits are overwritten. +# Read-only artifact store: Garage's web endpoint, bound to localhost. +server { + server_name {{ oats_public_hostname }}; + + # ODM outputs are large; do not buffer whole objects to disk. + proxy_buffering off; + client_max_body_size 0; + + location / { + proxy_pass {{ oats_web_endpoint }}; + proxy_set_header Host {{ oats_public_hostname }}; + } +} diff --git a/storage/deploy/storage.yml b/storage/deploy/storage.yml new file mode 100644 index 0000000..5e89387 --- /dev/null +++ b/storage/deploy/storage.yml @@ -0,0 +1,10 @@ +--- +# Deploy the OATS artifact store. See ../README.md. +# +# ansible-galaxy collection install -r requirements.yml +# ansible-playbook -i inventory.yml storage.yml +- name: OATS artifact store + hosts: all + become: true + roles: + - oats_storage From 6118d61d2f274d5303e820a3a1b366bf04249ab8 Mon Sep 17 00:00:00 2001 From: Michael Johnson Date: Wed, 19 Aug 2026 14:30:54 +0100 Subject: [PATCH 03/10] Move the cluster setup into Ansible and split the role by concern MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit init.sh was the last of the provisioning left in shell, which meant the cluster setup had two homes and the playbook shelled out to a script to do half its work. Its steps are all guarded Garage CLI calls, so they port directly to tasks: wait for the node, assign and apply the layout, create the bucket, enable anonymous web access, issue the writer key and grant it. The credentials are parsed straight from "key create" rather than scraped from a script's stdout. Local development runs the same role through inventory.local.yml, which puts the store under ~/.local/share, so dropping the script costs it nothing and the setup has one definition. The task list is split by what it sets up — disk, service, cluster, credentials, access, proxy — leaving main.yml as a table of contents. Verified on a local Garage: a fresh deploy from an empty state, a re-run reporting no changes (with the public-hostname path exercised), a run published through the result, and a cross-origin read of index.json over the aliased hostname. --- storage/README.md | 34 +-- storage/deploy/inventory.local.yml | 12 ++ .../roles/oats_storage/defaults/main.yml | 14 +- .../roles/oats_storage/tasks/access.yml | 33 +++ .../roles/oats_storage/tasks/cluster.yml | 62 ++++++ .../roles/oats_storage/tasks/credentials.yml | 67 ++++++ .../deploy/roles/oats_storage/tasks/disk.yml | 13 ++ .../deploy/roles/oats_storage/tasks/main.yml | 197 +----------------- .../deploy/roles/oats_storage/tasks/proxy.yml | 33 +++ .../roles/oats_storage/tasks/service.yml | 62 ++++++ storage/init.sh | 70 ------- 11 files changed, 323 insertions(+), 274 deletions(-) create mode 100644 storage/deploy/inventory.local.yml create mode 100644 storage/deploy/roles/oats_storage/tasks/access.yml create mode 100644 storage/deploy/roles/oats_storage/tasks/cluster.yml create mode 100644 storage/deploy/roles/oats_storage/tasks/credentials.yml create mode 100644 storage/deploy/roles/oats_storage/tasks/disk.yml create mode 100644 storage/deploy/roles/oats_storage/tasks/proxy.yml create mode 100644 storage/deploy/roles/oats_storage/tasks/service.yml delete mode 100755 storage/init.sh diff --git a/storage/README.md b/storage/README.md index 8561501..0c388e9 100644 --- a/storage/README.md +++ b/storage/README.md @@ -41,11 +41,17 @@ enumerates the bucket: ## Deployment -`deploy/` holds an Ansible role that does the whole host setup: mounts the data -disk, deploys this directory, generates Garage's secrets, starts the container, -runs `init.sh`, applies CORS, and aliases the bucket to the public hostname -(optionally writing the nginx vhost). It is idempotent — re-run it to roll out a -config change. +`deploy/` holds an Ansible role that does the whole setup — there is no +provisioning left in shell. Its task files split along what they set up: + +- `disk.yml` — format and mount the data disk +- `service.yml` — deploy this directory, generate Garage's secrets, start the container +- `cluster.yml` — node layout, bucket, anonymous web access +- `credentials.yml` — issue the writer key and store it +- `access.yml` — install `mc`, apply CORS +- `proxy.yml` — alias the bucket to its hostname, nginx vhost + +It is idempotent — re-run it to roll out a config change. ```sh cd deploy @@ -76,20 +82,20 @@ rollback. ## Local development -Docker only — no Ansible needed: +The same role runs against this machine, putting the store under +`~/.local/share/oats-store`: ```sh -cp .env.example .env # fill GARAGE_RPC_SECRET, GARAGE_ADMIN_TOKEN -docker compose up -d -./init.sh # idempotent; prints the writer key once -mc alias set oats http://127.0.0.1:3900 -mc cors set oats/oats-runs cors.xml +cd deploy +ansible-playbook -i inventory.local.yml storage.yml ``` +Credentials land in `~/.local/share/oats-store/conf/writer.env`. + Garage resolves a bucket from the request `Host`, matched against a bucket -global alias. `init.sh` creates the alias `oats-runs`; to reach the web endpoint -(`127.0.0.1:3902`) from a browser, point that name at `127.0.0.1` in -`/etc/hosts`, or add an alias for whatever hostname you use. +global alias — the bucket's own name, `oats-runs`, is one. To reach the web +endpoint (`127.0.0.1:3902`) from a browser, point that name at `127.0.0.1` in +`/etc/hosts`, or set `oats_public_hostname` to the name you want and re-run. ## Publishing diff --git a/storage/deploy/inventory.local.yml b/storage/deploy/inventory.local.yml new file mode 100644 index 0000000..97f07c3 --- /dev/null +++ b/storage/deploy/inventory.local.yml @@ -0,0 +1,12 @@ +--- +# Local development: run the store on this machine, under ~/.local/share. +# ansible-playbook -i inventory.local.yml storage.yml +all: + hosts: + localhost: + ansible_connection: local + ansible_become: false + oats_storage_dir: "{{ ansible_env.HOME }}/.local/share/oats-store" + oats_data_root: "{{ ansible_env.HOME }}/.local/share/oats-store/garage" + oats_conf_dir: "{{ ansible_env.HOME }}/.local/share/oats-store/conf" + oats_mc_path: "{{ ansible_env.HOME }}/.local/bin/mc" diff --git a/storage/deploy/roles/oats_storage/defaults/main.yml b/storage/deploy/roles/oats_storage/defaults/main.yml index 4880ae8..74553bb 100644 --- a/storage/deploy/roles/oats_storage/defaults/main.yml +++ b/storage/deploy/roles/oats_storage/defaults/main.yml @@ -1,6 +1,6 @@ --- -# Where the committed store config (compose.yaml, garage.toml, cors.xml, -# init.sh) is deployed to on the host. +# Where the committed store config (compose.yaml, garage.toml, cors.xml) +# is deployed to on the host. oats_storage_dir: /opt/oats-storage # Garage's metadata and data live here, on the data disk. @@ -13,11 +13,17 @@ oats_data_device: "" oats_data_fstype: ext4 oats_bucket: oats-runs +oats_key_name: oats-writer +oats_zone: dc1 +oats_capacity: 100G + +# The Garage CLI lives inside the container; RUST_LOG quietens its chatter. +oats_garage: docker compose exec -T -e RUST_LOG=error garage /garage oats_s3_endpoint: http://127.0.0.1:3900 oats_web_endpoint: http://127.0.0.1:3902 -# Credentials written here, root-only. The writer secret is shown once by -# init.sh, so this file is the only copy on the host. +# Credentials written here, root-only. Garage reveals a secret only when the +# key is created, so this file is the only copy on the host. oats_conf_dir: /etc/oats oats_writer_env: "{{ oats_conf_dir }}/writer.env" diff --git a/storage/deploy/roles/oats_storage/tasks/access.yml b/storage/deploy/roles/oats_storage/tasks/access.yml new file mode 100644 index 0000000..c32721d --- /dev/null +++ b/storage/deploy/roles/oats_storage/tasks/access.yml @@ -0,0 +1,33 @@ +--- +- name: Create the directory mc is installed into + ansible.builtin.file: + path: "{{ oats_mc_path | dirname }}" + state: directory + mode: "0755" + +- name: Install the mc S3 client + ansible.builtin.get_url: + url: "{{ oats_mc_url }}" + dest: "{{ oats_mc_path }}" + mode: "0755" + +- name: Register the bucket with mc + ansible.builtin.command: >- + {{ oats_mc_path }} --config-dir {{ oats_conf_dir }}/mc alias set oats + {{ oats_s3_endpoint }} {{ oats_key_id }} {{ oats_key_secret }} + changed_when: false + no_log: true + +- name: Read the current CORS configuration + ansible.builtin.command: >- + {{ oats_mc_path }} --config-dir {{ oats_conf_dir }}/mc + cors get oats/{{ oats_bucket }} + register: oats_cors_now + changed_when: false + failed_when: false + +- name: Allow cross-origin reads + ansible.builtin.command: >- + {{ oats_mc_path }} --config-dir {{ oats_conf_dir }}/mc cors set oats/{{ oats_bucket }} + {{ oats_storage_dir }}/cors.xml + when: "'AllowedOrigin' not in oats_cors_now.stdout" diff --git a/storage/deploy/roles/oats_storage/tasks/cluster.yml b/storage/deploy/roles/oats_storage/tasks/cluster.yml new file mode 100644 index 0000000..6b32553 --- /dev/null +++ b/storage/deploy/roles/oats_storage/tasks/cluster.yml @@ -0,0 +1,62 @@ +--- +- name: Wait for Garage to accept commands + ansible.builtin.command: "{{ oats_garage }} status" + args: + chdir: "{{ oats_storage_dir }}" + register: oats_status + until: oats_status.rc == 0 + retries: 30 + delay: 1 + changed_when: false + +- name: Give the node a layout role + when: "'NO ROLE ASSIGNED' in oats_status.stdout" + block: + - name: Stage the node's role + ansible.builtin.command: >- + {{ oats_garage }} layout assign -z {{ oats_zone }} -c {{ oats_capacity }} + {{ (oats_status.stdout_lines | select('search', 'NO ROLE ASSIGNED') | first).split() | first }} + args: + chdir: "{{ oats_storage_dir }}" + + - name: Read the staged layout version + ansible.builtin.command: "{{ oats_garage }} layout show" + args: + chdir: "{{ oats_storage_dir }}" + register: oats_layout + changed_when: false + + - name: Apply the staged layout + ansible.builtin.command: >- + {{ oats_garage }} layout apply --version + {{ (oats_layout.stdout | regex_search('layout version: (\d+)', '\1') | first | int) + 1 }} + args: + chdir: "{{ oats_storage_dir }}" + +- name: Look for the bucket + ansible.builtin.command: "{{ oats_garage }} bucket info {{ oats_bucket }}" + args: + chdir: "{{ oats_storage_dir }}" + register: oats_bucket_check + changed_when: false + failed_when: false + +- name: Create the bucket + ansible.builtin.command: "{{ oats_garage }} bucket create {{ oats_bucket }}" + args: + chdir: "{{ oats_storage_dir }}" + when: oats_bucket_check.rc != 0 + +- name: Read the bucket's state + ansible.builtin.command: "{{ oats_garage }} bucket info {{ oats_bucket }}" + args: + chdir: "{{ oats_storage_dir }}" + register: oats_bucket_state + changed_when: false + +# Anonymous read is served by Garage's web endpoint, enabled per bucket. +- name: Serve the bucket's objects anonymously + ansible.builtin.command: "{{ oats_garage }} bucket website --allow {{ oats_bucket }}" + args: + chdir: "{{ oats_storage_dir }}" + when: "'Website access: true' not in oats_bucket_state.stdout" diff --git a/storage/deploy/roles/oats_storage/tasks/credentials.yml b/storage/deploy/roles/oats_storage/tasks/credentials.yml new file mode 100644 index 0000000..d95cdef --- /dev/null +++ b/storage/deploy/roles/oats_storage/tasks/credentials.yml @@ -0,0 +1,67 @@ +--- +- name: Look for the writer key + ansible.builtin.command: "{{ oats_garage }} key info {{ oats_key_name }}" + args: + chdir: "{{ oats_storage_dir }}" + register: oats_key_check + changed_when: false + failed_when: false + no_log: true + +- name: Create the writer key + ansible.builtin.command: "{{ oats_garage }} key create {{ oats_key_name }}" + args: + chdir: "{{ oats_storage_dir }}" + register: oats_key_new + when: oats_key_check.rc != 0 + no_log: true + +- name: Store the writer credentials + ansible.builtin.copy: + content: | + AWS_ACCESS_KEY_ID={{ oats_key_new.stdout | regex_search('Key ID: (\S+)', '\1') | first }} + AWS_SECRET_ACCESS_KEY={{ oats_key_new.stdout | regex_search('Secret key: (\S+)', '\1') | first }} + dest: "{{ oats_writer_env }}" + mode: "0600" + when: oats_key_check.rc != 0 + no_log: true + +- name: Grant the writer key access to the bucket + ansible.builtin.command: >- + {{ oats_garage }} bucket allow --read --write --owner + {{ oats_bucket }} --key {{ oats_key_name }} + args: + chdir: "{{ oats_storage_dir }}" + when: oats_bucket_state.stdout is not search('RWO\s+\S+\s+' ~ oats_key_name) + +- name: Check that the writer credentials are on disk + ansible.builtin.stat: + path: "{{ oats_writer_env }}" + register: oats_writer_stat + +- name: Fail when the writer secret is unrecoverable + ansible.builtin.fail: + msg: >- + The writer key exists in Garage but {{ oats_writer_env }} is missing, and + Garage only reveals a secret at creation time. Delete the key + ("garage key delete oats-writer") and re-run to issue a new one. + when: not oats_writer_stat.stat.exists + +- name: Read the writer credentials + ansible.builtin.slurp: + src: "{{ oats_writer_env }}" + register: oats_writer_raw + no_log: true + +- name: Load the writer credentials + ansible.builtin.set_fact: + oats_key_id: "{{ oats_writer_raw.content | b64decode | regex_search('AWS_ACCESS_KEY_ID=(\\S+)', '\\1') | first }}" + oats_key_secret: "{{ oats_writer_raw.content | b64decode | regex_search('AWS_SECRET_ACCESS_KEY=(\\S+)', '\\1') | first }}" + no_log: true + +- name: Copy the writer credentials to the control node + ansible.builtin.fetch: + src: "{{ oats_writer_env }}" + dest: "{{ oats_fetch_dest }}" + flat: true + when: oats_fetch_writer_env | bool diff --git a/storage/deploy/roles/oats_storage/tasks/disk.yml b/storage/deploy/roles/oats_storage/tasks/disk.yml new file mode 100644 index 0000000..dd0e00f --- /dev/null +++ b/storage/deploy/roles/oats_storage/tasks/disk.yml @@ -0,0 +1,13 @@ +--- +- name: Create a filesystem on the data device + community.general.filesystem: + fstype: "{{ oats_data_fstype }}" + dev: "{{ oats_data_device }}" + +- name: Mount the data device + ansible.posix.mount: + path: "{{ oats_data_mount }}" + src: "{{ oats_data_device }}" + fstype: "{{ oats_data_fstype }}" + opts: defaults,nofail + state: mounted diff --git a/storage/deploy/roles/oats_storage/tasks/main.yml b/storage/deploy/roles/oats_storage/tasks/main.yml index f30e43d..9542c90 100644 --- a/storage/deploy/roles/oats_storage/tasks/main.yml +++ b/storage/deploy/roles/oats_storage/tasks/main.yml @@ -6,196 +6,21 @@ failed_when: oats_docker_check.rc != 0 - name: Prepare the data disk + ansible.builtin.import_tasks: disk.yml when: oats_data_device | length > 0 - block: - - name: Create a filesystem on the data device - community.general.filesystem: - fstype: "{{ oats_data_fstype }}" - dev: "{{ oats_data_device }}" - - name: Mount the data device - ansible.posix.mount: - path: "{{ oats_data_mount }}" - src: "{{ oats_data_device }}" - fstype: "{{ oats_data_fstype }}" - opts: defaults,nofail - state: mounted +- name: Deploy and start the Garage service + ansible.builtin.import_tasks: service.yml -- name: Create the store directories - ansible.builtin.file: - path: "{{ item.path }}" - state: directory - mode: "{{ item.mode }}" - loop: - - { path: "{{ oats_storage_dir }}", mode: "0755" } - - { path: "{{ oats_data_root }}/meta", mode: "0700" } - - { path: "{{ oats_data_root }}/data", mode: "0700" } - - { path: "{{ oats_conf_dir }}", mode: "0700" } - - { path: "{{ oats_conf_dir }}/mc", mode: "0700" } +- name: Set up the cluster and its bucket + ansible.builtin.import_tasks: cluster.yml -- name: Deploy the store configuration - ansible.builtin.copy: - src: "{{ playbook_dir }}/../{{ item.name }}" - dest: "{{ oats_storage_dir }}/{{ item.name }}" - mode: "{{ item.mode }}" - loop: - - { name: compose.yaml, mode: "0644" } - - { name: garage.toml, mode: "0644" } - - { name: cors.xml, mode: "0644" } - - { name: init.sh, mode: "0755" } - - { name: publish_run.sh, mode: "0755" } - notify: Restart Garage +- name: Issue the writer key + ansible.builtin.import_tasks: credentials.yml -- name: Point Garage's volumes at the data disk - ansible.builtin.template: - src: compose.override.yaml.j2 - dest: "{{ oats_storage_dir }}/compose.override.yaml" - mode: "0644" - notify: Restart Garage +- name: Open the bucket for browser reads + ansible.builtin.import_tasks: access.yml -# Garage's own secrets are generated once and never rotated by a re-run. -- name: Look for existing Garage secrets - ansible.builtin.stat: - path: "{{ oats_storage_dir }}/.env" - register: oats_env_file - -- name: Generate Garage secrets - ansible.builtin.set_fact: - oats_rpc_secret: "{{ lookup('ansible.builtin.password', '/dev/null', chars=['hexdigits'], length=64) | lower }}" - oats_admin_token: "{{ lookup('ansible.builtin.password', '/dev/null', length=32) }}" - when: not oats_env_file.stat.exists - no_log: true - -- name: Write the Garage secrets file - ansible.builtin.copy: - content: | - GARAGE_RPC_SECRET={{ oats_rpc_secret }} - GARAGE_ADMIN_TOKEN={{ oats_admin_token }} - dest: "{{ oats_storage_dir }}/.env" - mode: "0600" - when: not oats_env_file.stat.exists - no_log: true - -- name: Start Garage - community.docker.docker_compose_v2: - project_src: "{{ oats_storage_dir }}" - state: present - -- name: Flush handlers so Garage restarts before it is initialised - ansible.builtin.meta: flush_handlers - -# init.sh is idempotent and is the same script used for local development, so -# the layout / bucket / website / key setup has exactly one definition. -- name: Initialise the cluster, bucket and writer key - ansible.builtin.command: ./init.sh - args: - chdir: "{{ oats_storage_dir }}" - register: oats_init - changed_when: "'Writer key created' in oats_init.stdout" - no_log: true - -- name: Store the writer credentials - ansible.builtin.copy: - content: | - AWS_ACCESS_KEY_ID={{ oats_init.stdout | regex_search('AWS_ACCESS_KEY_ID=(\S+)', '\1') | first }} - AWS_SECRET_ACCESS_KEY={{ oats_init.stdout | regex_search('AWS_SECRET_ACCESS_KEY=(\S+)', '\1') | first }} - dest: "{{ oats_writer_env }}" - mode: "0600" - when: "'Writer key created' in oats_init.stdout" - no_log: true - -- name: Check that the writer credentials are on disk - ansible.builtin.stat: - path: "{{ oats_writer_env }}" - register: oats_writer_stat - -- name: Fail when the writer secret is unrecoverable - ansible.builtin.fail: - msg: >- - The writer key exists in Garage but {{ oats_writer_env }} is missing, and - Garage only reveals a secret at creation time. Delete the key - ("garage key delete oats-writer") and re-run to issue a new one. - when: not oats_writer_stat.stat.exists - -- name: Read the writer credentials - ansible.builtin.slurp: - src: "{{ oats_writer_env }}" - register: oats_writer_raw - no_log: true - -- name: Load the writer credentials - ansible.builtin.set_fact: - oats_key_id: "{{ oats_writer_raw.content | b64decode | regex_search('AWS_ACCESS_KEY_ID=(\\S+)', '\\1') | first }}" - oats_key_secret: "{{ oats_writer_raw.content | b64decode | regex_search('AWS_SECRET_ACCESS_KEY=(\\S+)', '\\1') | first }}" - no_log: true - -- name: Install the mc S3 client - ansible.builtin.get_url: - url: "{{ oats_mc_url }}" - dest: "{{ oats_mc_path }}" - mode: "0755" - -- name: Register the bucket with mc - ansible.builtin.command: >- - {{ oats_mc_path }} --config-dir {{ oats_conf_dir }}/mc alias set oats - {{ oats_s3_endpoint }} {{ oats_key_id }} {{ oats_key_secret }} - changed_when: false - no_log: true - -- name: Read the current CORS configuration - ansible.builtin.command: >- - {{ oats_mc_path }} --config-dir {{ oats_conf_dir }}/mc - cors get oats/{{ oats_bucket }} - register: oats_cors_now - changed_when: false - failed_when: false - -- name: Allow cross-origin reads - ansible.builtin.command: >- - {{ oats_mc_path }} --config-dir {{ oats_conf_dir }}/mc cors set oats/{{ oats_bucket }} - {{ oats_storage_dir }}/cors.xml - when: "'AllowedOrigin' not in oats_cors_now.stdout" - -- name: Serve the bucket under the public hostname +- name: Publish the bucket under its hostname + ansible.builtin.import_tasks: proxy.yml when: oats_public_hostname | length > 0 - block: - # Garage resolves a bucket from the request Host, so the public hostname - # has to exist as a bucket alias. - - name: Read the bucket's aliases - ansible.builtin.command: docker compose exec -T garage /garage bucket info {{ oats_bucket }} - args: - chdir: "{{ oats_storage_dir }}" - register: oats_bucket_info - changed_when: false - - - name: Alias the bucket to the public hostname - ansible.builtin.command: >- - docker compose exec -T garage /garage bucket alias - {{ oats_bucket }} {{ oats_public_hostname }} - args: - chdir: "{{ oats_storage_dir }}" - when: oats_public_hostname not in oats_bucket_info.stdout - - - name: Write the nginx vhost - when: oats_nginx_manage | bool - block: - - name: Install the store's server block - ansible.builtin.template: - src: nginx-store.conf.j2 - dest: "{{ oats_nginx_site_dir }}/oats-store" - mode: "0644" - notify: Reload nginx - - - name: Enable the store's server block - ansible.builtin.file: - src: "{{ oats_nginx_site_dir }}/oats-store" - dest: "{{ oats_nginx_enabled_dir }}/oats-store" - state: link - notify: Reload nginx - -- name: Copy the writer credentials to the control node - ansible.builtin.fetch: - src: "{{ oats_writer_env }}" - dest: "{{ oats_fetch_dest }}" - flat: true - when: oats_fetch_writer_env | bool diff --git a/storage/deploy/roles/oats_storage/tasks/proxy.yml b/storage/deploy/roles/oats_storage/tasks/proxy.yml new file mode 100644 index 0000000..3b91bd2 --- /dev/null +++ b/storage/deploy/roles/oats_storage/tasks/proxy.yml @@ -0,0 +1,33 @@ +--- +# Garage resolves a bucket from the request Host, so the public hostname +# has to exist as a bucket alias. +- name: Read the bucket's aliases + ansible.builtin.command: "{{ oats_garage }} bucket info {{ oats_bucket }}" + args: + chdir: "{{ oats_storage_dir }}" + register: oats_bucket_info + changed_when: false + +- name: Alias the bucket to the public hostname + ansible.builtin.command: >- + {{ oats_garage }} bucket alias {{ oats_bucket }} {{ oats_public_hostname }} + args: + chdir: "{{ oats_storage_dir }}" + when: oats_public_hostname not in oats_bucket_info.stdout + +- name: Write the nginx vhost + when: oats_nginx_manage | bool + block: + - name: Install the store's server block + ansible.builtin.template: + src: nginx-store.conf.j2 + dest: "{{ oats_nginx_site_dir }}/oats-store" + mode: "0644" + notify: Reload nginx + + - name: Enable the store's server block + ansible.builtin.file: + src: "{{ oats_nginx_site_dir }}/oats-store" + dest: "{{ oats_nginx_enabled_dir }}/oats-store" + state: link + notify: Reload nginx diff --git a/storage/deploy/roles/oats_storage/tasks/service.yml b/storage/deploy/roles/oats_storage/tasks/service.yml new file mode 100644 index 0000000..b09d700 --- /dev/null +++ b/storage/deploy/roles/oats_storage/tasks/service.yml @@ -0,0 +1,62 @@ +--- +- name: Create the store directories + ansible.builtin.file: + path: "{{ item.path }}" + state: directory + mode: "{{ item.mode }}" + loop: + - { path: "{{ oats_storage_dir }}", mode: "0755" } + - { path: "{{ oats_data_root }}/meta", mode: "0700" } + - { path: "{{ oats_data_root }}/data", mode: "0700" } + - { path: "{{ oats_conf_dir }}", mode: "0700" } + - { path: "{{ oats_conf_dir }}/mc", mode: "0700" } + +- name: Deploy the store configuration + ansible.builtin.copy: + src: "{{ playbook_dir }}/../{{ item.name }}" + dest: "{{ oats_storage_dir }}/{{ item.name }}" + mode: "{{ item.mode }}" + loop: + - { name: compose.yaml, mode: "0644" } + - { name: garage.toml, mode: "0644" } + - { name: cors.xml, mode: "0644" } + - { name: publish_run.sh, mode: "0755" } + notify: Restart Garage + +- name: Point Garage's volumes at the data disk + ansible.builtin.template: + src: compose.override.yaml.j2 + dest: "{{ oats_storage_dir }}/compose.override.yaml" + mode: "0644" + notify: Restart Garage + +# Garage's own secrets are generated once and never rotated by a re-run. +- name: Look for existing Garage secrets + ansible.builtin.stat: + path: "{{ oats_storage_dir }}/.env" + register: oats_env_file + +- name: Generate Garage secrets + ansible.builtin.set_fact: + oats_rpc_secret: "{{ lookup('ansible.builtin.password', '/dev/null', chars=['hexdigits'], length=64) | lower }}" + oats_admin_token: "{{ lookup('ansible.builtin.password', '/dev/null', length=32) }}" + when: not oats_env_file.stat.exists + no_log: true + +- name: Write the Garage secrets file + ansible.builtin.copy: + content: | + GARAGE_RPC_SECRET={{ oats_rpc_secret }} + GARAGE_ADMIN_TOKEN={{ oats_admin_token }} + dest: "{{ oats_storage_dir }}/.env" + mode: "0600" + when: not oats_env_file.stat.exists + no_log: true + +- name: Start Garage + community.docker.docker_compose_v2: + project_src: "{{ oats_storage_dir }}" + state: present + +- name: Flush handlers so Garage restarts before it is initialised + ansible.builtin.meta: flush_handlers diff --git a/storage/init.sh b/storage/init.sh deleted file mode 100755 index 982121c..0000000 --- a/storage/init.sh +++ /dev/null @@ -1,70 +0,0 @@ -#!/bin/bash -# -# One-time initialisation of the Garage cluster used to store OATS runs. -# Idempotent: safe to re-run. Performs the Garage-CLI setup (single-node -# layout, bucket, website hosting, writer key). The CORS step needs an S3 -# client and is documented separately in README.md. -# -# Run from the storage/ directory after `docker compose up -d`: -# ./init.sh -# -# Prints the writer key credentials at the end. Store them in an env file / -# CI secret (see README) -- never commit them. - -set -euo pipefail - -BUCKET="${OATS_BUCKET:-oats-runs}" -KEY_NAME="${OATS_KEY_NAME:-oats-writer}" -CAPACITY="${OATS_CAPACITY:-100G}" -ZONE="${OATS_ZONE:-dc1}" - -g() { docker compose exec -T -e RUST_LOG=error garage /garage "$@"; } - -echo "Waiting for Garage to be reachable..." -for _ in $(seq 1 30); do - if g status >/dev/null 2>&1; then break; fi - sleep 1 -done - -# Assign a layout to the single node if it has no role yet. -node=$(g status 2>/dev/null | awk '/NO ROLE ASSIGNED/{print $1}') -if [ -n "$node" ]; then - echo "Assigning layout to node $node ($CAPACITY, zone $ZONE)..." - g layout assign -z "$ZONE" -c "$CAPACITY" "$node" >/dev/null - ver=$(g layout show 2>/dev/null | awk '/Current cluster layout version/{print $NF}') - g layout apply --version "$((ver + 1))" >/dev/null - echo "Layout applied." -else - echo "Node already has a role; skipping layout assignment." -fi - -# Create the bucket and enable public website (anonymous read) hosting. -if ! g bucket info "$BUCKET" >/dev/null 2>&1; then - echo "Creating bucket $BUCKET..." - g bucket create "$BUCKET" >/dev/null -fi -g bucket website --allow "$BUCKET" >/dev/null -echo "Bucket $BUCKET ready with website (anonymous read) enabled." - -# Create the writer key if it does not exist and grant it access. -if g key info "$KEY_NAME" >/dev/null 2>&1; then - echo "Key $KEY_NAME already exists; not recreating (its secret is only shown at creation)." - g bucket allow --read --write --owner "$BUCKET" --key "$KEY_NAME" >/dev/null -else - echo "Creating writer key $KEY_NAME..." - created=$(g key create "$KEY_NAME" 2>/dev/null) - g bucket allow --read --write --owner "$BUCKET" --key "$KEY_NAME" >/dev/null - kid=$(echo "$created" | awk -F': ' '/Key ID/{print $2}') - ksec=$(echo "$created" | awk -F': ' '/Secret key/{print $2}') - echo - echo "==========================================================" - echo "Writer key created -- store these, they are shown ONCE:" - echo " AWS_ACCESS_KEY_ID=$kid" - echo " AWS_SECRET_ACCESS_KEY=$ksec" - echo "==========================================================" -fi - -echo -echo "Next: apply CORS so browsers can read runs (see README):" -echo " mc alias set oats http://localhost:3900 " -echo " mc cors set oats/$BUCKET cors.xml" From f6cf19cfda7f930d918729d4dc184dbaf0e4f7c4 Mon Sep 17 00:00:00 2001 From: Michael Johnson Date: Wed, 19 Aug 2026 15:07:19 +0100 Subject: [PATCH 04/10] Describe the store as it stands in docs and comments The docs still carried the shape of earlier iterations: the top-level README advertised a filesystem publish target that no longer exists, the store contract diagram kept a level the harness has since dropped from run directories, and .env.example described hand-filling a secrets file the deploy role now generates and nothing referenced. Rewrite the prose to state what the store does rather than how it got here, correct the run layout to tests///, and drop the orphaned example file. --- README.md | 5 ++--- storage/.env.example | 11 ----------- storage/README.md | 17 ++++++++--------- storage/compose.yaml | 2 +- storage/deploy/inventory.example.yml | 4 ++-- storage/garage.toml | 5 ++--- 6 files changed, 15 insertions(+), 29 deletions(-) delete mode 100644 storage/.env.example diff --git a/README.md b/README.md index 6cf9c41..e4d1a86 100644 --- a/README.md +++ b/README.md @@ -121,10 +121,9 @@ This command will copy all odm_orthophoto.tif files from all test cases into `/m ### Publishing runs to durable storage `results/runs/` is local and prunable. To retain runs and serve them read-only -over HTTP for the comparison viewer, publish them to the artifact store — a -Garage (S3-compatible) bucket or a plain filesystem root — with +over HTTP for the comparison viewer, publish them to the artifact store with `storage/publish_run.sh`. See [`storage/README.md`](storage/README.md) for the -backend, setup, and publish workflow. +store's layout, setup and publish workflow. ## Roadmap diff --git a/storage/.env.example b/storage/.env.example deleted file mode 100644 index b6e7312..0000000 --- a/storage/.env.example +++ /dev/null @@ -1,11 +0,0 @@ -# Copy to storage/.env and fill in. storage/.env is git-ignored; never commit it. -# -# Generate fresh secrets with: -# GARAGE_RPC_SECRET -> openssl rand -hex 32 -# GARAGE_ADMIN_TOKEN -> openssl rand -base64 32 - -# Shared secret authenticating RPC between Garage nodes (required even single-node). -GARAGE_RPC_SECRET= - -# Bearer token for the Garage admin API on :3903. -GARAGE_ADMIN_TOKEN= diff --git a/storage/README.md b/storage/README.md index 0c388e9..8f7e522 100644 --- a/storage/README.md +++ b/storage/README.md @@ -21,7 +21,7 @@ oats-runs (bucket root) /// # one self-contained run run_manifest.json reports/*.xml - ///... # the ODM outputs + tests///... # the ODM outputs ``` The key is the run directory's own relative path under `results/runs/` — read @@ -41,8 +41,8 @@ enumerates the bucket: ## Deployment -`deploy/` holds an Ansible role that does the whole setup — there is no -provisioning left in shell. Its task files split along what they set up: +`deploy/` holds an Ansible role that performs the entire setup. Its task files +split along what they set up: - `disk.yml` — format and mount the data disk - `service.yml` — deploy this directory, generate Garage's secrets, start the container @@ -65,8 +65,7 @@ The writer credentials are generated on first run and left in to copy them back for pasting into the CI secret store. Garage only reveals a secret at creation, so if that file is lost the key has to be reissued. -Two things stay manual, on the Proxmox host — they are one-shot and outside the -VM: +Two operations are manual, on the Proxmox host — one-shot, and outside the VM: ```sh qm set -scsi1 /dev/disk/by-id/ # pass the data disk through @@ -76,9 +75,8 @@ qm set -onboot 1 # store is always-on Snapshot before a Garage upgrade (`qm snapshot pre-upgrade`) for instant rollback. -> Status: the role is verified end-to-end against a local Garage (fresh deploy, -> idempotent re-run, publish, cross-origin read). Running it against the -> Proxmox VM is the one remaining step, pending access to that host. +> The role has not yet been run against the Proxmox VM; that is the remaining +> deployment step. ## Local development @@ -106,7 +104,8 @@ See `--help`. ## Credentials -- `GARAGE_RPC_SECRET` / `GARAGE_ADMIN_TOKEN`: `storage/.env`, read by `compose.yaml`. +- `GARAGE_RPC_SECRET` / `GARAGE_ADMIN_TOKEN`: generated on first deploy into + `.env` beside `compose.yaml`, which reads them. - Writer key/secret: `/etc/oats/writer.env` on the host, and the CI secret store. - The viewer needs none — it reads the public endpoint only. diff --git a/storage/compose.yaml b/storage/compose.yaml index 9ef6456..3edf425 100644 --- a/storage/compose.yaml +++ b/storage/compose.yaml @@ -5,7 +5,7 @@ services: restart: unless-stopped network_mode: host environment: - # Secrets are injected from storage/.env (never committed). See README. + # Read from .env in this directory, written by the deploy role. GARAGE_RPC_SECRET: ${GARAGE_RPC_SECRET:?set GARAGE_RPC_SECRET in storage/.env} GARAGE_ADMIN_TOKEN: ${GARAGE_ADMIN_TOKEN:?set GARAGE_ADMIN_TOKEN in storage/.env} volumes: diff --git a/storage/deploy/inventory.example.yml b/storage/deploy/inventory.example.yml index 4c1595d..036d9f0 100644 --- a/storage/deploy/inventory.example.yml +++ b/storage/deploy/inventory.example.yml @@ -16,6 +16,6 @@ all: # matching alias so Garage resolves it by virtual host. # oats_public_hostname: oats-store.example.com - # Let the playbook write the nginx vhost too. Leave false if the viewer's - # task owns the proxy config. + # Let the playbook write the nginx vhost. Leave false when the proxy + # config is managed elsewhere. # oats_nginx_manage: true diff --git a/storage/garage.toml b/storage/garage.toml index 8910c73..fac4508 100644 --- a/storage/garage.toml +++ b/storage/garage.toml @@ -4,9 +4,8 @@ db_engine = "lmdb" replication_factor = 1 -# Everything binds to localhost. Public read is served only through the host's -# reverse proxy (see README), which reaches the web endpoint over 127.0.0.1 — so -# nothing is exposed on the network directly. +# Every endpoint binds to loopback, so nothing is reachable off the host +# directly; the reverse proxy fronts the web endpoint for public read. rpc_bind_addr = "127.0.0.1:3901" rpc_public_addr = "127.0.0.1:3901" From db56311f09c74ba4dcdcd8763d13238096e37183 Mon Sep 17 00:00:00 2001 From: Michael Johnson Date: Wed, 19 Aug 2026 15:12:48 +0100 Subject: [PATCH 05/10] Tighten the storage docs and comments Cut the prose down to what a reader needs: the store contract, the deployment command, and the facts that are not obvious from the code. The tree and index.json shapes stay, since the viewer codes against them, as does publish_run.sh's usage text, which is its --help output. --- README.md | 7 +- storage/README.md | 99 ++++++++----------- storage/compose.yaml | 2 +- storage/deploy/inventory.example.yml | 15 ++- storage/deploy/inventory.local.yml | 2 +- .../roles/oats_storage/defaults/main.yml | 19 ++-- .../roles/oats_storage/tasks/cluster.yml | 2 +- .../deploy/roles/oats_storage/tasks/proxy.yml | 4 +- .../roles/oats_storage/tasks/service.yml | 2 +- .../templates/compose.override.yaml.j2 | 2 +- .../templates/nginx-store.conf.j2 | 6 +- storage/deploy/storage.yml | 3 - storage/garage.toml | 3 +- storage/publish_run.sh | 7 +- 14 files changed, 69 insertions(+), 104 deletions(-) diff --git a/README.md b/README.md index e4d1a86..c11acce 100644 --- a/README.md +++ b/README.md @@ -120,10 +120,9 @@ This command will copy all odm_orthophoto.tif files from all test cases into `/m ### Publishing runs to durable storage -`results/runs/` is local and prunable. To retain runs and serve them read-only -over HTTP for the comparison viewer, publish them to the artifact store with -`storage/publish_run.sh`. See [`storage/README.md`](storage/README.md) for the -store's layout, setup and publish workflow. +`results/runs/` is local and prunable. `storage/publish_run.sh` publishes a run +to the artifact store, which serves it read-only for the comparison viewer. See +[`storage/README.md`](storage/README.md). ## Roadmap diff --git a/storage/README.md b/storage/README.md index 8f7e522..cace6f9 100644 --- a/storage/README.md +++ b/storage/README.md @@ -1,19 +1,16 @@ # OATS run artifact storage -Durable, addressable storage for OATS run directories, served read-only over -HTTP(S) so the comparison viewer can enumerate and load runs. +Durable storage for OATS run directories, served read-only over HTTP(S) for the +comparison viewer. -**Backend: [Garage](https://garagehq.deuxfleurs.fr/)** — an S3-compatible object -store that ships as a single Rust binary (`dxflrs/garage`), run single-node with -`replication_factor = 1`. Chosen over the archived MinIO CE and a plain -nginx-over-a-directory: the S3 API lets the backend be swapped or grown -(multi-node / offsite) with only an endpoint change. CI and the viewer depend -only on the contract below, never on Garage itself. +**Backend: [Garage](https://garagehq.deuxfleurs.fr/)** — S3-compatible, a single +Rust binary, single-node with `replication_factor = 1`. Chosen over the archived +MinIO CE and nginx-over-a-directory for its S3 API, so it can be swapped or +grown later. CI and the viewer depend on the contract below, not on Garage. ## Store contract -The store mirrors the harness layout — no re-keying at publish time. Each run -sits at its identity path with a top-level `index.json` next to it: +Each run sits at its identity path, exactly as the harness writes it: ``` oats-runs (bucket root) @@ -24,11 +21,8 @@ oats-runs (bucket root) tests///... # the ODM outputs ``` -The key is the run directory's own relative path under `results/runs/` — read -straight from `run_manifest.json`'s `run_key`. - -`index.json` is the only file the viewer reads to discover runs; it never -enumerates the bucket: +The key is the manifest's `run_key`. `index.json` is all the viewer reads to +find runs; it never lists the bucket: ```json { "generated": "...", "runs": [ { @@ -37,83 +31,72 @@ enumerates the bucket: "datasets_total": 2, "passed": 0, "failed": 2 } ] } ``` -`passed`/`failed` aggregate the per-dataset bats exit codes. +`passed`/`failed` aggregate per-dataset bats exit codes. ## Deployment -`deploy/` holds an Ansible role that performs the entire setup. Its task files -split along what they set up: - -- `disk.yml` — format and mount the data disk -- `service.yml` — deploy this directory, generate Garage's secrets, start the container -- `cluster.yml` — node layout, bucket, anonymous web access -- `credentials.yml` — issue the writer key and store it -- `access.yml` — install `mc`, apply CORS -- `proxy.yml` — alias the bucket to its hostname, nginx vhost +`deploy/` holds an idempotent Ansible role; re-run it for any change: -It is idempotent — re-run it to roll out a config change. +- `disk.yml` — data disk +- `service.yml` — config, secrets, container +- `cluster.yml` — layout, bucket, anonymous read +- `credentials.yml` — writer key +- `access.yml` — `mc`, CORS +- `proxy.yml` — hostname alias, nginx vhost ```sh cd deploy ansible-galaxy collection install -r requirements.yml -cp inventory.example.yml inventory.yml # set the host, disk and hostname +cp inventory.example.yml inventory.yml # host, disk, hostname ansible-playbook storage.yml ``` -The writer credentials are generated on first run and left in -`/etc/oats/writer.env` on the host (root-only); pass `-e oats_fetch_writer_env=true` -to copy them back for pasting into the CI secret store. Garage only reveals a -secret at creation, so if that file is lost the key has to be reissued. +Writer credentials land in root-only `/etc/oats/writer.env`; +`-e oats_fetch_writer_env=true` copies them back for CI. Garage reveals a secret +only at creation. -Two operations are manual, on the Proxmox host — one-shot, and outside the VM: +Manual, one-shot, on the Proxmox host: ```sh qm set -scsi1 /dev/disk/by-id/ # pass the data disk through -qm set -onboot 1 # store is always-on +qm set -onboot 1 ``` -Snapshot before a Garage upgrade (`qm snapshot pre-upgrade`) for instant -rollback. +Snapshot before a Garage upgrade: `qm snapshot pre-upgrade`. -> The role has not yet been run against the Proxmox VM; that is the remaining -> deployment step. +> Not yet run against the Proxmox VM; that is the remaining deployment step. ## Local development -The same role runs against this machine, putting the store under -`~/.local/share/oats-store`: +The same role against this machine, into `~/.local/share/oats-store`: ```sh cd deploy ansible-playbook -i inventory.local.yml storage.yml ``` -Credentials land in `~/.local/share/oats-store/conf/writer.env`. - -Garage resolves a bucket from the request `Host`, matched against a bucket -global alias — the bucket's own name, `oats-runs`, is one. To reach the web -endpoint (`127.0.0.1:3902`) from a browser, point that name at `127.0.0.1` in -`/etc/hosts`, or set `oats_public_hostname` to the name you want and re-run. +Garage resolves the bucket from the request `Host`. To browse +`127.0.0.1:3902`, point `oats-runs` at `127.0.0.1` in `/etc/hosts`. ## Publishing -`./publish_run.sh s3://oats-runs --endpoint http://127.0.0.1:3900` -writes a run and regenerates `index.json`. Idempotent per run key; needs `mc` -and `jq`. Credentials come from `AWS_ACCESS_KEY_ID` / `AWS_SECRET_ACCESS_KEY`. -See `--help`. +```sh +./publish_run.sh s3://oats-runs --endpoint http://127.0.0.1:3900 +``` + +Idempotent per run key. Needs `mc`, `jq`, and `AWS_ACCESS_KEY_ID` / +`AWS_SECRET_ACCESS_KEY`. See `--help`. ## Credentials -- `GARAGE_RPC_SECRET` / `GARAGE_ADMIN_TOKEN`: generated on first deploy into - `.env` beside `compose.yaml`, which reads them. -- Writer key/secret: `/etc/oats/writer.env` on the host, and the CI secret store. -- The viewer needs none — it reads the public endpoint only. +- `GARAGE_RPC_SECRET` / `GARAGE_ADMIN_TOKEN`: generated on first deploy into `.env`. +- Writer key: `/etc/oats/writer.env` and the CI secret store. +- The viewer needs none. -Nothing secret is committed; `.env`, `compose.override.yaml`, `deploy/inventory.yml` -and `deploy/writer.env` are git-ignored. +`.env`, `compose.override.yaml`, `deploy/inventory.yml` and `deploy/writer.env` +are git-ignored. ## Durability -Single node, single disk. Runs are re-derivable (re-run CI), or sync the bucket -offsite with `rclone sync`. Back up the guest's config with `vzdump`; don't -bother capturing the bulk data disk. +Single node, single disk. Runs are re-derivable by re-running CI; sync offsite +with `rclone sync`. Back up the guest with `vzdump`. diff --git a/storage/compose.yaml b/storage/compose.yaml index 3edf425..d1bc8ea 100644 --- a/storage/compose.yaml +++ b/storage/compose.yaml @@ -5,7 +5,7 @@ services: restart: unless-stopped network_mode: host environment: - # Read from .env in this directory, written by the deploy role. + # From .env here, written by the deploy role. GARAGE_RPC_SECRET: ${GARAGE_RPC_SECRET:?set GARAGE_RPC_SECRET in storage/.env} GARAGE_ADMIN_TOKEN: ${GARAGE_ADMIN_TOKEN:?set GARAGE_ADMIN_TOKEN in storage/.env} volumes: diff --git a/storage/deploy/inventory.example.yml b/storage/deploy/inventory.example.yml index 036d9f0..d30b296 100644 --- a/storage/deploy/inventory.example.yml +++ b/storage/deploy/inventory.example.yml @@ -1,21 +1,18 @@ --- -# Copy to inventory.yml (git-ignored) and edit. One host: the always-on VM that -# runs CI, Garage and the viewer. +# Copy to inventory.yml (git-ignored). One host: the VM running CI, Garage +# and the viewer. all: hosts: oats-store: ansible_host: oats-vm.lan ansible_user: michael - # Block device passed through from the Proxmox host. Leave unset to skip - # all disk handling and use whatever is already mounted at oats_data_root. - # WARNING: when set and unformatted, the playbook formats this device. + # Passed through from the Proxmox host; unset skips all disk handling. + # WARNING: an unformatted device is formatted. # oats_data_device: /dev/sdb - # Public hostname the store is served under. When set, the bucket gains a - # matching alias so Garage resolves it by virtual host. + # Served under this name; the bucket gains a matching alias. # oats_public_hostname: oats-store.example.com - # Let the playbook write the nginx vhost. Leave false when the proxy - # config is managed elsewhere. + # Write the nginx vhost. False when the proxy is managed elsewhere. # oats_nginx_manage: true diff --git a/storage/deploy/inventory.local.yml b/storage/deploy/inventory.local.yml index 97f07c3..19c4a18 100644 --- a/storage/deploy/inventory.local.yml +++ b/storage/deploy/inventory.local.yml @@ -1,5 +1,5 @@ --- -# Local development: run the store on this machine, under ~/.local/share. +# Local development, under ~/.local/share: # ansible-playbook -i inventory.local.yml storage.yml all: hosts: diff --git a/storage/deploy/roles/oats_storage/defaults/main.yml b/storage/deploy/roles/oats_storage/defaults/main.yml index 74553bb..eecb857 100644 --- a/storage/deploy/roles/oats_storage/defaults/main.yml +++ b/storage/deploy/roles/oats_storage/defaults/main.yml @@ -1,14 +1,12 @@ --- -# Where the committed store config (compose.yaml, garage.toml, cors.xml) -# is deployed to on the host. +# Where compose.yaml, garage.toml and cors.xml are deployed. oats_storage_dir: /opt/oats-storage -# Garage's metadata and data live here, on the data disk. +# Garage's metadata and data, on the data disk. oats_data_mount: /data oats_data_root: "{{ oats_data_mount }}/oats-garage" -# Optional block device to format and mount at oats_data_mount. Unset means -# the playbook touches no disks. +# Formatted and mounted at oats_data_mount. Unset means no disks are touched. oats_data_device: "" oats_data_fstype: ext4 @@ -17,26 +15,23 @@ oats_key_name: oats-writer oats_zone: dc1 oats_capacity: 100G -# The Garage CLI lives inside the container; RUST_LOG quietens its chatter. +# The CLI lives in the container; RUST_LOG quietens it. oats_garage: docker compose exec -T -e RUST_LOG=error garage /garage oats_s3_endpoint: http://127.0.0.1:3900 oats_web_endpoint: http://127.0.0.1:3902 -# Credentials written here, root-only. Garage reveals a secret only when the -# key is created, so this file is the only copy on the host. +# Root-only. Garage reveals a secret only at creation, so this is the only copy. oats_conf_dir: /etc/oats oats_writer_env: "{{ oats_conf_dir }}/writer.env" -# Copy the writer credentials back to the control node (for pasting into the -# CI secret store). Off by default so secrets do not spread silently. +# Copy the credentials to the control node, for the CI secret store. oats_fetch_writer_env: false oats_fetch_dest: ./writer.env oats_mc_path: /usr/local/bin/mc oats_mc_url: https://dl.min.io/client/mc/release/linux-amd64/mc -# Public hostname; when set the bucket is aliased to it so Garage resolves it -# by virtual host. nginx vhost is only written when oats_nginx_manage is true. +# When set, the bucket is aliased to it. The vhost needs oats_nginx_manage. oats_public_hostname: "" oats_nginx_manage: false oats_nginx_site_dir: /etc/nginx/sites-available diff --git a/storage/deploy/roles/oats_storage/tasks/cluster.yml b/storage/deploy/roles/oats_storage/tasks/cluster.yml index 6b32553..dcd3663 100644 --- a/storage/deploy/roles/oats_storage/tasks/cluster.yml +++ b/storage/deploy/roles/oats_storage/tasks/cluster.yml @@ -54,7 +54,7 @@ register: oats_bucket_state changed_when: false -# Anonymous read is served by Garage's web endpoint, enabled per bucket. +# Anonymous read is served by the web endpoint, enabled per bucket. - name: Serve the bucket's objects anonymously ansible.builtin.command: "{{ oats_garage }} bucket website --allow {{ oats_bucket }}" args: diff --git a/storage/deploy/roles/oats_storage/tasks/proxy.yml b/storage/deploy/roles/oats_storage/tasks/proxy.yml index 3b91bd2..8a6d8da 100644 --- a/storage/deploy/roles/oats_storage/tasks/proxy.yml +++ b/storage/deploy/roles/oats_storage/tasks/proxy.yml @@ -1,6 +1,6 @@ --- -# Garage resolves a bucket from the request Host, so the public hostname -# has to exist as a bucket alias. +# Garage resolves the bucket from the request Host, so the hostname must +# exist as an alias. - name: Read the bucket's aliases ansible.builtin.command: "{{ oats_garage }} bucket info {{ oats_bucket }}" args: diff --git a/storage/deploy/roles/oats_storage/tasks/service.yml b/storage/deploy/roles/oats_storage/tasks/service.yml index b09d700..8700649 100644 --- a/storage/deploy/roles/oats_storage/tasks/service.yml +++ b/storage/deploy/roles/oats_storage/tasks/service.yml @@ -30,7 +30,7 @@ mode: "0644" notify: Restart Garage -# Garage's own secrets are generated once and never rotated by a re-run. +# Generated once; a re-run never rotates them. - name: Look for existing Garage secrets ansible.builtin.stat: path: "{{ oats_storage_dir }}/.env" diff --git a/storage/deploy/roles/oats_storage/templates/compose.override.yaml.j2 b/storage/deploy/roles/oats_storage/templates/compose.override.yaml.j2 index 153a591..92cdfe9 100644 --- a/storage/deploy/roles/oats_storage/templates/compose.override.yaml.j2 +++ b/storage/deploy/roles/oats_storage/templates/compose.override.yaml.j2 @@ -1,4 +1,4 @@ -# Managed by Ansible (roles/oats_storage). Local edits are overwritten. +# Managed by Ansible; local edits are overwritten. services: garage: volumes: diff --git a/storage/deploy/roles/oats_storage/templates/nginx-store.conf.j2 b/storage/deploy/roles/oats_storage/templates/nginx-store.conf.j2 index d962cfa..8635204 100644 --- a/storage/deploy/roles/oats_storage/templates/nginx-store.conf.j2 +++ b/storage/deploy/roles/oats_storage/templates/nginx-store.conf.j2 @@ -1,9 +1,9 @@ -# Managed by Ansible (roles/oats_storage). Local edits are overwritten. -# Read-only artifact store: Garage's web endpoint, bound to localhost. +# Managed by Ansible; local edits are overwritten. +# Read-only store: Garage's web endpoint on loopback. server { server_name {{ oats_public_hostname }}; - # ODM outputs are large; do not buffer whole objects to disk. + # ODM outputs are large; do not buffer them to disk. proxy_buffering off; client_max_body_size 0; diff --git a/storage/deploy/storage.yml b/storage/deploy/storage.yml index 5e89387..468d239 100644 --- a/storage/deploy/storage.yml +++ b/storage/deploy/storage.yml @@ -1,8 +1,5 @@ --- # Deploy the OATS artifact store. See ../README.md. -# -# ansible-galaxy collection install -r requirements.yml -# ansible-playbook -i inventory.yml storage.yml - name: OATS artifact store hosts: all become: true diff --git a/storage/garage.toml b/storage/garage.toml index fac4508..b29049f 100644 --- a/storage/garage.toml +++ b/storage/garage.toml @@ -4,8 +4,7 @@ db_engine = "lmdb" replication_factor = 1 -# Every endpoint binds to loopback, so nothing is reachable off the host -# directly; the reverse proxy fronts the web endpoint for public read. +# Loopback only; the reverse proxy fronts the web endpoint for public read. rpc_bind_addr = "127.0.0.1:3901" rpc_public_addr = "127.0.0.1:3901" diff --git a/storage/publish_run.sh b/storage/publish_run.sh index 4531702..2fb6604 100755 --- a/storage/publish_run.sh +++ b/storage/publish_run.sh @@ -1,12 +1,7 @@ #!/bin/bash # # Publish an OATS run directory to the Garage (S3) store and regenerate the -# store's index.json, using mc (MinIO client) against the S3 endpoint. -# -# The store mirrors the harness layout: a run keyed by its identity -# (//) lives at the bucket root next to a -# top-level index.json listing every run. Re-publishing the same run overwrites -# it cleanly, so the operation is idempotent per run key. +# store's index.json. Idempotent per run key. See storage/README.md. # # Usage: # publish_run.sh --endpoint URL From 0d3c13040ba77c3a453d91a05eb06abba068e498 Mon Sep 17 00:00:00 2001 From: Michael Johnson Date: Wed, 19 Aug 2026 15:53:24 +0100 Subject: [PATCH 06/10] Protect index.json and read the writer key back from Garage MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A failed enumeration could replace a populated index.json with an empty one: mc find's stderr went to /dev/null, jq turned no input into an empty run list, and mc pipe wrote it before pipefail reported the failure. The index is now assembled locally and checked before upload — non-empty, and containing the run just published — and the enumeration's errors are left visible. --reindex rebuilds the index from the manifests already in the store, which are the source of truth. The writer key is read back from Garage with "key info --show-secret" rather than captured at creation, so /etc/oats/writer.env is a cache that is rewritten when it goes missing instead of a file whose loss forced the key to be reissued. Verified against a local Garage: publish and --reindex both report the run count; a rebuild that enumerates nothing refuses and leaves a populated index untouched; deleting writer.env and re-running restores the identical credentials. --- storage/README.md | 52 ++++++++----- .../roles/oats_storage/tasks/credentials.yml | 50 +++++-------- storage/publish_run.sh | 73 +++++++++++++++---- 3 files changed, 111 insertions(+), 64 deletions(-) diff --git a/storage/README.md b/storage/README.md index cace6f9..7839ad2 100644 --- a/storage/README.md +++ b/storage/README.md @@ -1,12 +1,10 @@ # OATS run artifact storage -Durable storage for OATS run directories, served read-only over HTTP(S) for the -comparison viewer. +Durable storage for OATS run directories, served read-only over HTTP(S). -**Backend: [Garage](https://garagehq.deuxfleurs.fr/)** — S3-compatible, a single -Rust binary, single-node with `replication_factor = 1`. Chosen over the archived -MinIO CE and nginx-over-a-directory for its S3 API, so it can be swapped or -grown later. CI and the viewer depend on the contract below, not on Garage. +**Backend: [Garage](https://garagehq.deuxfleurs.fr/)** — S3-compatible API which +means we keep the storage contract separate from the instance and can swap out +the backend later if we need more storage etc. ## Store contract @@ -14,7 +12,7 @@ Each run sits at its identity path, exactly as the harness writes it: ``` oats-runs (bucket root) - index.json # every run, newest first + index.json # every run, newest first /// # one self-contained run run_manifest.json reports/*.xml @@ -25,10 +23,20 @@ The key is the manifest's `run_key`. `index.json` is all the viewer reads to find runs; it never lists the bucket: ```json -{ "generated": "...", "runs": [ { - "key": "...", "timestamp": "...", "image": "...", "image_digest": "...", - "odm_git_revision": "...", "group": "...", - "datasets_total": 2, "passed": 0, "failed": 2 } ] } +{ + "generated": "...", + "runs": [ { + "key": "...", + "timestamp": "...", + "image": "...", + "image_digest": "...", + "odm_git_revision": "...", + "group": "...", + "datasets_total": 2, + "passed": 0, + "failed": 2 + } ] +} ``` `passed`/`failed` aggregate per-dataset bats exit codes. @@ -47,15 +55,16 @@ find runs; it never lists the bucket: ```sh cd deploy ansible-galaxy collection install -r requirements.yml -cp inventory.example.yml inventory.yml # host, disk, hostname +cp inventory.example.yml inventory.yml # update host, disk, hostname ansible-playbook storage.yml ``` -Writer credentials land in root-only `/etc/oats/writer.env`; -`-e oats_fetch_writer_env=true` copies them back for CI. Garage reveals a secret -only at creation. +Garage holds the writer key and shows it on demand, so the deploy caches it in +root-only `/etc/oats/writer.env` and rewrites it if it goes missing. +`-e oats_fetch_writer_env=true` copies it back for CI. -Manual, one-shot, on the Proxmox host: +Manual, on the Proxmox host — it needs hypervisor root, and the VM's lifecycle +belongs to the runner setup rather than the store: ```sh qm set -scsi1 /dev/disk/by-id/ # pass the data disk through @@ -64,11 +73,9 @@ qm set -onboot 1 Snapshot before a Garage upgrade: `qm snapshot pre-upgrade`. -> Not yet run against the Proxmox VM; that is the remaining deployment step. - ## Local development -The same role against this machine, into `~/.local/share/oats-store`: +To run the storage role locally, into `~/.local/share/oats-store`: ```sh cd deploy @@ -87,12 +94,19 @@ Garage resolves the bucket from the request `Host`. To browse Idempotent per run key. Needs `mc`, `jq`, and `AWS_ACCESS_KEY_ID` / `AWS_SECRET_ACCESS_KEY`. See `--help`. +`index.json` is derived from the run manifests, so `--reindex` rebuilds it from +whatever is already in the store. + ## Credentials - `GARAGE_RPC_SECRET` / `GARAGE_ADMIN_TOKEN`: generated on first deploy into `.env`. - Writer key: `/etc/oats/writer.env` and the CI secret store. - The viewer needs none. +The writer key uses the standard S3 variable names, so any S3 tool reads it +without translation. Name the CI secrets for OATS and map them to those names in +the workflow. + `.env`, `compose.override.yaml`, `deploy/inventory.yml` and `deploy/writer.env` are git-ignored. diff --git a/storage/deploy/roles/oats_storage/tasks/credentials.yml b/storage/deploy/roles/oats_storage/tasks/credentials.yml index d95cdef..db92fc5 100644 --- a/storage/deploy/roles/oats_storage/tasks/credentials.yml +++ b/storage/deploy/roles/oats_storage/tasks/credentials.yml @@ -12,17 +12,6 @@ ansible.builtin.command: "{{ oats_garage }} key create {{ oats_key_name }}" args: chdir: "{{ oats_storage_dir }}" - register: oats_key_new - when: oats_key_check.rc != 0 - no_log: true - -- name: Store the writer credentials - ansible.builtin.copy: - content: | - AWS_ACCESS_KEY_ID={{ oats_key_new.stdout | regex_search('Key ID: (\S+)', '\1') | first }} - AWS_SECRET_ACCESS_KEY={{ oats_key_new.stdout | regex_search('Secret key: (\S+)', '\1') | first }} - dest: "{{ oats_writer_env }}" - mode: "0600" when: oats_key_check.rc != 0 no_log: true @@ -34,29 +23,30 @@ chdir: "{{ oats_storage_dir }}" when: oats_bucket_state.stdout is not search('RWO\s+\S+\s+' ~ oats_key_name) -- name: Check that the writer credentials are on disk - ansible.builtin.stat: - path: "{{ oats_writer_env }}" - register: oats_writer_stat - -- name: Fail when the writer secret is unrecoverable - ansible.builtin.fail: - msg: >- - The writer key exists in Garage but {{ oats_writer_env }} is missing, and - Garage only reveals a secret at creation time. Delete the key - ("garage key delete oats-writer") and re-run to issue a new one. - when: not oats_writer_stat.stat.exists - -- name: Read the writer credentials - ansible.builtin.slurp: - src: "{{ oats_writer_env }}" - register: oats_writer_raw +# Garage holds the secret and will show it on demand, so the key is read back +# from the cluster rather than captured at creation. +- name: Read the writer key + ansible.builtin.command: "{{ oats_garage }} key info --show-secret {{ oats_key_name }}" + args: + chdir: "{{ oats_storage_dir }}" + register: oats_key_info + changed_when: false no_log: true - name: Load the writer credentials ansible.builtin.set_fact: - oats_key_id: "{{ oats_writer_raw.content | b64decode | regex_search('AWS_ACCESS_KEY_ID=(\\S+)', '\\1') | first }}" - oats_key_secret: "{{ oats_writer_raw.content | b64decode | regex_search('AWS_SECRET_ACCESS_KEY=(\\S+)', '\\1') | first }}" + oats_key_id: "{{ oats_key_info.stdout | regex_search('Key ID: (\\S+)', '\\1') | first }}" + oats_key_secret: "{{ oats_key_info.stdout | regex_search('Secret key: (\\S+)', '\\1') | first }}" + no_log: true + +# A cache for anyone publishing by hand on the host; rewritten if it is lost. +- name: Store the writer credentials + ansible.builtin.copy: + content: | + AWS_ACCESS_KEY_ID={{ oats_key_id }} + AWS_SECRET_ACCESS_KEY={{ oats_key_secret }} + dest: "{{ oats_writer_env }}" + mode: "0600" no_log: true - name: Copy the writer credentials to the control node diff --git a/storage/publish_run.sh b/storage/publish_run.sh index 2fb6604..d709d4b 100755 --- a/storage/publish_run.sh +++ b/storage/publish_run.sh @@ -5,6 +5,7 @@ # # Usage: # publish_run.sh --endpoint URL +# publish_run.sh --reindex --endpoint URL # # A run directory containing run_manifest.json, e.g. # results/runs/// @@ -12,6 +13,8 @@ # # Options: # --endpoint URL S3 endpoint, e.g. http://127.0.0.1:3900 (required) +# --reindex Rebuild index.json from the manifests already in the +# store, without publishing a run # --mc PATH mc binary to use (default: mc on PATH, or $OATS_MC) # -h, --help Show this help # @@ -28,11 +31,13 @@ usage() { RUN_DIR="" TARGET="" ENDPOINT="" +REINDEX=NO MC="${OATS_MC:-mc}" while [ $# -gt 0 ]; do case "$1" in --endpoint) ENDPOINT="$2"; shift 2;; + --reindex) REINDEX=YES; shift;; --mc) MC="$2"; shift 2;; -h|--help) usage 0;; -*) echo "Unknown option: $1" >&2; usage 1;; @@ -45,7 +50,14 @@ while [ $# -gt 0 ]; do esac done -[ -n "$RUN_DIR" ] && [ -n "$TARGET" ] || usage 1 +# --reindex takes no run directory, so the sole positional is the target +if [ "$REINDEX" = YES ] && [ -z "$TARGET" ]; then + TARGET="$RUN_DIR" + RUN_DIR="" +fi + +[ -n "$TARGET" ] || usage 1 +[ "$REINDEX" = YES ] || [ -n "$RUN_DIR" ] || usage 1 case "$TARGET" in s3://*) ;; *) echo "Target must be an s3:// URL (e.g. s3://oats-runs)" >&2; usage 1;; @@ -57,17 +69,20 @@ esac command -v jq >/dev/null || { echo "jq is required" >&2; exit 1; } command -v "$MC" >/dev/null || { echo "mc client not found: $MC" >&2; exit 1; } -MANIFEST="$RUN_DIR/run_manifest.json" -[ -f "$MANIFEST" ] || { echo "No run_manifest.json in $RUN_DIR" >&2; exit 1; } +RUN_KEY="" +if [ "$REINDEX" = NO ]; then + MANIFEST="$RUN_DIR/run_manifest.json" + [ -f "$MANIFEST" ] || { echo "No run_manifest.json in $RUN_DIR" >&2; exit 1; } -# The run key is the run's identity path, written by the harness. Fall back to -# the run directory's own trailing // path if absent. -RUN_KEY=$(jq -r '.run_key // empty' "$MANIFEST") -if [ -z "$RUN_KEY" ]; then - abs=$(cd "$RUN_DIR" && pwd) - RUN_KEY=$(echo "$abs" | rev | cut -d/ -f1-3 | rev) + # The run key is the run's identity path, written by the harness. Fall back + # to the run directory's own trailing // path if absent. + RUN_KEY=$(jq -r '.run_key // empty' "$MANIFEST") + if [ -z "$RUN_KEY" ]; then + abs=$(cd "$RUN_DIR" && pwd) + RUN_KEY=$(echo "$abs" | rev | cut -d/ -f1-3 | rev) + fi + [ -n "$RUN_KEY" ] || { echo "Could not determine run key" >&2; exit 1; } fi -[ -n "$RUN_KEY" ] || { echo "Could not determine run key" >&2; exit 1; } # jq program turning a stream of run manifests into the store index. Each run # contributes its identity plus an aggregate pass/fail count derived from the @@ -97,11 +112,39 @@ trap 'rm -rf "$cfg"' EXIT "$MC" --config-dir "$cfg" alias set "$alias" "$ENDPOINT" \ "$AWS_ACCESS_KEY_ID" "$AWS_SECRET_ACCESS_KEY" >/dev/null +# The manifests in the store are the source of truth and index.json is derived +# from them, so it can be rebuilt at any time. It is assembled locally and +# checked before upload: a failed enumeration must not replace a populated +# index with a short or empty one. +reindex() { + local required="${1:-}" keys="$cfg/keys" manifests="$cfg/manifests" index="$cfg/index.json" + + "$MC" --config-dir "$cfg" find "$base" --name run_manifest.json > "$keys" + : > "$manifests" + while IFS= read -r key; do + "$MC" --config-dir "$cfg" cat "$key" >> "$manifests" + done < "$keys" + + jq -s "$INDEX_JQ" < "$manifests" > "$index" + INDEX_COUNT=$(jq '.runs | length' "$index") + + [ "$INDEX_COUNT" -gt 0 ] || { echo "Enumerated no runs; leaving index.json alone" >&2; exit 1; } + if [ -n "$required" ]; then + jq -e --arg k "$required" 'any(.runs[]; .key == $k)' "$index" >/dev/null \ + || { echo "$required missing from the rebuilt index; leaving index.json alone" >&2; exit 1; } + fi + + "$MC" --config-dir "$cfg" pipe "$base/index.json" < "$index" >/dev/null +} + +if [ "$REINDEX" = YES ]; then + reindex + echo "Rebuilt $TARGET/index.json ($INDEX_COUNT runs)" + exit 0 +fi + "$MC" --config-dir "$cfg" mirror --overwrite --remove "$RUN_DIR/" "$base/$RUN_KEY" >/dev/null -"$MC" --config-dir "$cfg" find "$base" --name run_manifest.json 2>/dev/null \ - | while IFS= read -r p; do "$MC" --config-dir "$cfg" cat "$p"; done \ - | jq -s "$INDEX_JQ" \ - | "$MC" --config-dir "$cfg" pipe "$base/index.json" >/dev/null +reindex "$RUN_KEY" echo "Published run $RUN_KEY to $TARGET/$RUN_KEY" -echo "Regenerated $TARGET/index.json" +echo "Regenerated $TARGET/index.json ($INDEX_COUNT runs)" From 78e976bb5ead4e7b8416273deac7428f79b0f313 Mon Sep 17 00:00:00 2001 From: Michael Johnson Date: Fri, 4 Sep 2026 16:14:16 +0100 Subject: [PATCH 07/10] Run Garage as a static binary and bootstrap it with --single-node Garage v2.3.0 replaces the docker compose stack: a checksummed binary under systemd, --single-node --default-bucket for the layout, bucket and writer key, and the admin API for website access and CORS. The role no longer formats disks, installs mc or needs Galaxy collections. The S3 API and web endpoint bind on all interfaces so CI and an external proxy can reach them; RPC and admin stay on loopback. A Molecule scenario runs the role in a systemd container and checks idempotence and the exposed ports. The workflow gains a publish step that is skipped until OATS_STORE_ENDPOINT is set. --- .github/workflows/oats.yml | 14 ++- docs/ci.md | 19 ++- storage/.gitignore | 3 - storage/README.md | 86 ++++++++----- storage/compose.yaml | 18 --- storage/cors.xml | 13 -- storage/deploy/inventory.example.yml | 18 ++- storage/deploy/inventory.local.yml | 12 -- storage/deploy/requirements.yml | 8 -- .../roles/oats_storage/defaults/main.yml | 52 ++++---- .../roles/oats_storage/handlers/main.yml | 5 +- .../molecule/default/converge.yml | 6 + .../molecule/default/molecule.yml | 17 +++ .../oats_storage/molecule/default/verify.yml | 47 +++++++ .../roles/oats_storage/tasks/access.yml | 33 ----- .../roles/oats_storage/tasks/bucket.yml | 29 +++++ .../roles/oats_storage/tasks/cluster.yml | 62 ---------- .../roles/oats_storage/tasks/credentials.yml | 57 --------- .../deploy/roles/oats_storage/tasks/disk.yml | 13 -- .../deploy/roles/oats_storage/tasks/main.yml | 38 +++--- .../deploy/roles/oats_storage/tasks/proxy.yml | 8 +- .../roles/oats_storage/tasks/service.yml | 116 ++++++++++++------ .../templates/compose.override.yaml.j2 | 6 - .../oats_storage/templates/garage.service.j2 | 20 +++ .../oats_storage/templates/garage.toml.j2 | 26 ++++ .../templates/nginx-store.conf.j2 | 4 +- storage/garage.toml | 22 ---- 27 files changed, 364 insertions(+), 388 deletions(-) delete mode 100644 storage/compose.yaml delete mode 100644 storage/cors.xml delete mode 100644 storage/deploy/inventory.local.yml delete mode 100644 storage/deploy/requirements.yml create mode 100644 storage/deploy/roles/oats_storage/molecule/default/converge.yml create mode 100644 storage/deploy/roles/oats_storage/molecule/default/molecule.yml create mode 100644 storage/deploy/roles/oats_storage/molecule/default/verify.yml delete mode 100644 storage/deploy/roles/oats_storage/tasks/access.yml create mode 100644 storage/deploy/roles/oats_storage/tasks/bucket.yml delete mode 100644 storage/deploy/roles/oats_storage/tasks/cluster.yml delete mode 100644 storage/deploy/roles/oats_storage/tasks/credentials.yml delete mode 100644 storage/deploy/roles/oats_storage/tasks/disk.yml delete mode 100644 storage/deploy/roles/oats_storage/templates/compose.override.yaml.j2 create mode 100644 storage/deploy/roles/oats_storage/templates/garage.service.j2 create mode 100644 storage/deploy/roles/oats_storage/templates/garage.toml.j2 delete mode 100644 storage/garage.toml diff --git a/.github/workflows/oats.yml b/.github/workflows/oats.yml index b8ab885..097be3d 100644 --- a/.github/workflows/oats.yml +++ b/.github/workflows/oats.yml @@ -147,9 +147,17 @@ jobs: require_tests: true fail_on_failure: true - # TODO: This will need updating once the self-hosted Garage storage - # service exists For now we just store the text artifacts as the whole - # results directory is far too big to store as an artifact. + # The whole run goes to the artifact store when one is configured; the + # GitHub artifact keeps the small text outputs close to the run page. + - name: Publish run to the store + if: always() && steps.rundir.outputs.run_dir != '' && vars.OATS_STORE_ENDPOINT != '' + env: + AWS_ACCESS_KEY_ID: ${{ secrets.OATS_STORE_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.OATS_STORE_SECRET_ACCESS_KEY }} + run: | + storage/publish_run.sh "${{ steps.rundir.outputs.run_dir }}" s3://oats-runs \ + --endpoint "${{ vars.OATS_STORE_ENDPOINT }}" + - name: Upload run artifact if: always() && steps.rundir.outputs.run_dir != '' uses: actions/upload-artifact@v7 diff --git a/docs/ci.md b/docs/ci.md index 3f66db3..d5d9998 100644 --- a/docs/ci.md +++ b/docs/ci.md @@ -10,7 +10,7 @@ outside it, namely the runner, the cross-repo token and the ODM-side trigger. Register a runner against this repository (*Settings → Actions → Runners*). The default `self-hosted` label is all the workflow asks for. -It needs docker, plus `git`, `wget`, `rsync`, `sed`, `unzip`, `jq`, and +It needs docker, plus `git`, `wget`, `rsync`, `sed`, `unzip`, `jq`, `mc`, and `gdalinfo` on `PATH`; `./run` bootstraps bats itself. Give it plenty of RAM — the suite is RAM-bound and the `all` group sets the ceiling. Each full run generates about 60GB of output right now so as a rough guide we should aim for @@ -65,6 +65,17 @@ fails and the ODM publish job goes red. ## Results -Currently we store the text outputs as a github artifact which will allow us to -review any failures. This will be replaced by the s3-compatible Garage server -once its set up. +Every run uploads its reports and small text outputs as a GitHub artifact, +kept for 14 days, which is enough to review a failure from the run page. + +The whole run, ODM outputs included, goes to the artifact store described in +[`storage/README.md`](../storage/README.md) when the store is configured. The +`Publish run to the store` step runs `storage/publish_run.sh`. To configure it, +under *Settings → Secrets and variables → Actions* on this repository: + +- Variable `OATS_STORE_ENDPOINT`: the store's S3 API, e.g. `http://oats-vm.lan:3900`. +- Secrets `OATS_STORE_ACCESS_KEY_ID` and `OATS_STORE_SECRET_ACCESS_KEY`: the + two values the storage playbook prints on its first run. + +The runner reaches the endpoint directly, so port 3900 on the store must be +open to it. The step needs `mc` and `jq` on the runner. diff --git a/storage/.gitignore b/storage/.gitignore index ea2e739..f1600dd 100644 --- a/storage/.gitignore +++ b/storage/.gitignore @@ -1,4 +1 @@ -.env -compose.override.yaml deploy/inventory.yml -deploy/writer.env diff --git a/storage/README.md b/storage/README.md index 7839ad2..47f6d31 100644 --- a/storage/README.md +++ b/storage/README.md @@ -19,8 +19,8 @@ oats-runs (bucket root) tests///... # the ODM outputs ``` -The key is the manifest's `run_key`. `index.json` is all the viewer reads to -find runs; it never lists the bucket: +Each run is stored under its `run_key` from `run_manifest.json`. `index.json` is +all the viewer reads to find runs: ```json { @@ -43,52 +43,75 @@ find runs; it never lists the bucket: ## Deployment -`deploy/` holds an idempotent Ansible role; re-run it for any change: +`deploy/` holds an idempotent Ansible role; re-run it for any change. The +target needs systemd and ssh; the role installs the rest: -- `disk.yml` — data disk -- `service.yml` — config, secrets, container -- `cluster.yml` — layout, bucket, anonymous read -- `credentials.yml` — writer key -- `access.yml` — `mc`, CORS +- `service.yml` — `garage` binary, secrets, config, systemd unit +- `bucket.yml` — anonymous read, CORS - `proxy.yml` — hostname alias, nginx vhost ```sh cd deploy -ansible-galaxy collection install -r requirements.yml -cp inventory.example.yml inventory.yml # update host, disk, hostname +cp inventory.example.yml inventory.yml # update host, user, hostname ansible-playbook storage.yml ``` -Garage holds the writer key and shows it on demand, so the deploy caches it in -root-only `/etc/oats/writer.env` and rewrites it if it goes missing. -`-e oats_fetch_writer_env=true` copies it back for CI. +Data lives in `/var/lib/garage` by default but can be changed in +`inventory.yml`. The `oats_public_hostname` must match the nginx vhost. + +The S3 API listens on port 3900 and the web endpoint on 3902, on every +interface; RPC and the admin API stay on loopback. A reverse proxy on another +machine serves the store with: + +eg. +```nginx +server { + server_name oats-store.opendronemap.org; + location / { + proxy_pass http://:3902; + proxy_set_header Host $host; # Garage picks the bucket by Host + proxy_buffering off; # ODM outputs are large + ... + } +} +``` -Manual, on the Proxmox host — it needs hypervisor root, and the VM's lifecycle -belongs to the runner setup rather than the store: +Set `oats_public_hostname` to that `server_name`: the role aliases the bucket to +it, which is how Garage maps the domain to the bucket. Set `oats_nginx_manage` +only when nginx runs on the store VM itself. -```sh -qm set -scsi1 /dev/disk/by-id/ # pass the data disk through -qm set -onboot 1 -``` +Garage starts with `--single-node --default-bucket`, which assigns the layout, +creates the bucket and imports the writer key from `/etc/oats/writer.env`. The +role generates that key on the first run and prints it at the end, for the CI +secret store. Afterwards, `sudo cat /etc/oats/writer.env` on the host. + +Garage runs as the `garage` user under systemd, configured by +`/etc/garage.toml`; `garage status` on the host shows the node. To upgrade, snapshot the guest +(`qm snapshot pre-upgrade`), -Snapshot before a Garage upgrade: `qm snapshot pre-upgrade`. +To update versions, set `oats_garage_version` and `oats_garage_sha256` to the +new release and re-run the playbook. -## Local development +## Testing -To run the storage role locally, into `~/.local/share/oats-store`: +The role has a [Molecule](https://ansible.readthedocs.io/projects/molecule/) +scenario that applies it to a systemd container, applies it again to check +nothing changes, then checks the bucket is served anonymously with CORS: ```sh -cd deploy -ansible-playbook -i inventory.local.yml storage.yml +uv tool install molecule --with 'molecule-plugins[docker]' --with ansible-core +~/.local/share/uv/tools/molecule/bin/ansible-galaxy collection install community.docker +cd deploy/roles/oats_storage +molecule test ``` -Garage resolves the bucket from the request `Host`. To browse -`127.0.0.1:3902`, point `oats-runs` at `127.0.0.1` in `/etc/hosts`. +`molecule converge` leaves the container running for inspection; +`molecule destroy` removes it. ## Publishing ```sh -./publish_run.sh s3://oats-runs --endpoint http://127.0.0.1:3900 +./publish_run.sh s3://oats-runs --endpoint http://:3900 ``` Idempotent per run key. Needs `mc`, `jq`, and `AWS_ACCESS_KEY_ID` / @@ -99,16 +122,17 @@ whatever is already in the store. ## Credentials -- `GARAGE_RPC_SECRET` / `GARAGE_ADMIN_TOKEN`: generated on first deploy into `.env`. -- Writer key: `/etc/oats/writer.env` and the CI secret store. +- `rpc_secret`, `admin_token`: generated on first deploy into `/etc/oats`, + readable only by the `garage` user. The admin API listens on loopback + port 3903. +- Writer key: `/etc/oats/writer.env`, root-only, and the CI secret store. - The viewer needs none. The writer key uses the standard S3 variable names, so any S3 tool reads it without translation. Name the CI secrets for OATS and map them to those names in the workflow. -`.env`, `compose.override.yaml`, `deploy/inventory.yml` and `deploy/writer.env` -are git-ignored. +`deploy/inventory.yml` is git-ignored. ## Durability diff --git a/storage/compose.yaml b/storage/compose.yaml deleted file mode 100644 index d1bc8ea..0000000 --- a/storage/compose.yaml +++ /dev/null @@ -1,18 +0,0 @@ -services: - garage: - image: dxflrs/garage:v1.0.1 - container_name: oats-garage - restart: unless-stopped - network_mode: host - environment: - # From .env here, written by the deploy role. - GARAGE_RPC_SECRET: ${GARAGE_RPC_SECRET:?set GARAGE_RPC_SECRET in storage/.env} - GARAGE_ADMIN_TOKEN: ${GARAGE_ADMIN_TOKEN:?set GARAGE_ADMIN_TOKEN in storage/.env} - volumes: - - ./garage.toml:/etc/garage.toml:ro - - meta:/var/lib/garage/meta - - data:/var/lib/garage/data - -volumes: - meta: - data: diff --git a/storage/cors.xml b/storage/cors.xml deleted file mode 100644 index 221da39..0000000 --- a/storage/cors.xml +++ /dev/null @@ -1,13 +0,0 @@ - - - - * - GET - HEAD - * - ETag - Content-Length - Content-Type - 3600 - - diff --git a/storage/deploy/inventory.example.yml b/storage/deploy/inventory.example.yml index d30b296..9ff673d 100644 --- a/storage/deploy/inventory.example.yml +++ b/storage/deploy/inventory.example.yml @@ -1,18 +1,16 @@ --- -# Copy to inventory.yml (git-ignored). One host: the VM running CI, Garage -# and the viewer. +# Copy to inventory.yml (git-ignored) all: hosts: oats-store: - ansible_host: oats-vm.lan - ansible_user: michael + ansible_host: oats-vm.lan # VM hostname or IP + ansible_user: ubuntu # SSH username (needs sudo) - # Passed through from the Proxmox host; unset skips all disk handling. - # WARNING: an unformatted device is formatted. - # oats_data_device: /dev/sdb + # Set the nginx vhost so the store is accessible via HTTP. + oats_public_hostname: oats-store.opendronemap.org - # Served under this name; the bucket gains a matching alias. - # oats_public_hostname: oats-store.example.com + # Change if you want to store the data somewhere else. + # oats_data_root: /var/lib/garage - # Write the nginx vhost. False when the proxy is managed elsewhere. + # Install nginx alongside garage if there is no external proxy. # oats_nginx_manage: true diff --git a/storage/deploy/inventory.local.yml b/storage/deploy/inventory.local.yml deleted file mode 100644 index 19c4a18..0000000 --- a/storage/deploy/inventory.local.yml +++ /dev/null @@ -1,12 +0,0 @@ ---- -# Local development, under ~/.local/share: -# ansible-playbook -i inventory.local.yml storage.yml -all: - hosts: - localhost: - ansible_connection: local - ansible_become: false - oats_storage_dir: "{{ ansible_env.HOME }}/.local/share/oats-store" - oats_data_root: "{{ ansible_env.HOME }}/.local/share/oats-store/garage" - oats_conf_dir: "{{ ansible_env.HOME }}/.local/share/oats-store/conf" - oats_mc_path: "{{ ansible_env.HOME }}/.local/bin/mc" diff --git a/storage/deploy/requirements.yml b/storage/deploy/requirements.yml deleted file mode 100644 index e6d2e76..0000000 --- a/storage/deploy/requirements.yml +++ /dev/null @@ -1,8 +0,0 @@ ---- -collections: - - name: ansible.posix - version: ">=1.5.0" - - name: community.docker - version: ">=3.6.0" - - name: community.general - version: ">=8.0.0" diff --git a/storage/deploy/roles/oats_storage/defaults/main.yml b/storage/deploy/roles/oats_storage/defaults/main.yml index eecb857..8a9c613 100644 --- a/storage/deploy/roles/oats_storage/defaults/main.yml +++ b/storage/deploy/roles/oats_storage/defaults/main.yml @@ -1,35 +1,43 @@ --- -# Where compose.yaml, garage.toml and cors.xml are deployed. +# Where publish_run.sh is deployed. oats_storage_dir: /opt/oats-storage -# Garage's metadata and data, on the data disk. -oats_data_mount: /data -oats_data_root: "{{ oats_data_mount }}/oats-garage" - -# Formatted and mounted at oats_data_mount. Unset means no disks are touched. -oats_data_device: "" -oats_data_fstype: ext4 +# Garage's metadata and data. Mount a separate disk here first if wanted. +oats_data_root: /var/lib/garage oats_bucket: oats-runs -oats_key_name: oats-writer -oats_zone: dc1 -oats_capacity: 100G -# The CLI lives in the container; RUST_LOG quietens it. -oats_garage: docker compose exec -T -e RUST_LOG=error garage /garage -oats_s3_endpoint: http://127.0.0.1:3900 +# The bucket's public face: served by the web endpoint, readable from any origin. +oats_bucket_public: + websiteAccess: + enabled: true + indexDocument: index.html + corsRules: + - AllowedOrigin: ["*"] + AllowedMethod: [GET, HEAD] + AllowedHeader: ["*"] + ExposeHeader: [ETag, Content-Length, Content-Type] + MaxAgeSeconds: 3600 + +# The S3 API (3900) and web endpoint (3902) listen here; RPC and admin stay on +# loopback. +oats_bind_addr: 0.0.0.0 oats_web_endpoint: http://127.0.0.1:3902 +oats_admin_endpoint: http://127.0.0.1:3903 -# Root-only. Garage reveals a secret only at creation, so this is the only copy. -oats_conf_dir: /etc/oats -oats_writer_env: "{{ oats_conf_dir }}/writer.env" +# garage is installed here, which the role puts on PATH. +oats_bin_dir: /usr/local/bin +oats_garage_version: v2.3.0 +oats_garage_url: "https://garagehq.deuxfleurs.fr/_releases/{{ oats_garage_version }}/x86_64-unknown-linux-musl/garage" +# sha256sum of the download; Garage publishes none, so compute it on a bump. +oats_garage_sha256: f98d317942bb341151a2775162016bb50cf86b865d0108de03eb5db16e2120cd -# Copy the credentials to the control node, for the CI secret store. -oats_fetch_writer_env: false -oats_fetch_dest: ./writer.env +# Garage runs under systemd as this user. +oats_service_user: garage -oats_mc_path: /usr/local/bin/mc -oats_mc_url: https://dl.min.io/client/mc/release/linux-amd64/mc +# Secrets, generated on the first run and printed once. +oats_conf_dir: /etc/oats +oats_writer_env: "{{ oats_conf_dir }}/writer.env" # When set, the bucket is aliased to it. The vhost needs oats_nginx_manage. oats_public_hostname: "" diff --git a/storage/deploy/roles/oats_storage/handlers/main.yml b/storage/deploy/roles/oats_storage/handlers/main.yml index 1f05d17..dea975d 100644 --- a/storage/deploy/roles/oats_storage/handlers/main.yml +++ b/storage/deploy/roles/oats_storage/handlers/main.yml @@ -1,7 +1,8 @@ --- - name: Restart Garage - community.docker.docker_compose_v2: - project_src: "{{ oats_storage_dir }}" + ansible.builtin.systemd: + name: garage + daemon_reload: true state: restarted - name: Reload nginx diff --git a/storage/deploy/roles/oats_storage/molecule/default/converge.yml b/storage/deploy/roles/oats_storage/molecule/default/converge.yml new file mode 100644 index 0000000..3661c3e --- /dev/null +++ b/storage/deploy/roles/oats_storage/molecule/default/converge.yml @@ -0,0 +1,6 @@ +--- +- name: Converge + hosts: all + become: true + roles: + - oats_storage diff --git a/storage/deploy/roles/oats_storage/molecule/default/molecule.yml b/storage/deploy/roles/oats_storage/molecule/default/molecule.yml new file mode 100644 index 0000000..e43af11 --- /dev/null +++ b/storage/deploy/roles/oats_storage/molecule/default/molecule.yml @@ -0,0 +1,17 @@ +--- +# Run the role in a systemd container: molecule test +driver: + name: docker +platforms: + - name: oats-store + image: geerlingguy/docker-ubuntu2404-ansible:latest + command: "" + pre_build_image: true + privileged: true + cgroupns_mode: host + volumes: + - /sys/fs/cgroup:/sys/fs/cgroup:rw +provisioner: + name: ansible + env: + ANSIBLE_ROLES_PATH: ${MOLECULE_PROJECT_DIRECTORY}/.. diff --git a/storage/deploy/roles/oats_storage/molecule/default/verify.yml b/storage/deploy/roles/oats_storage/molecule/default/verify.yml new file mode 100644 index 0000000..b95f73b --- /dev/null +++ b/storage/deploy/roles/oats_storage/molecule/default/verify.yml @@ -0,0 +1,47 @@ +--- +- name: Verify + hosts: all + become: true + tasks: + - name: Read the bucket through the admin API + ansible.builtin.shell: >- + curl -sS --fail -H "Authorization: Bearer $(cat /etc/oats/admin_token)" + "http://127.0.0.1:3903/v2/GetBucketInfo?globalAlias=oats-runs" + register: bucket + changed_when: false + + - name: Check the bucket is served anonymously with CORS + ansible.builtin.assert: + that: + - (bucket.stdout | from_json).websiteAccess + - (bucket.stdout | from_json).corsRules | length == 1 + - (bucket.stdout | from_json)['keys'] | length == 1 + + - name: Fetch a missing object from the web endpoint + ansible.builtin.uri: + url: http://127.0.0.1:3902/nothing + headers: + Host: oats-runs + Origin: http://example.com + status_code: 404 + register: web + + - name: Check the web endpoint answers with CORS headers + ansible.builtin.assert: + that: web.access_control_allow_origin == '*' + + - name: Reach the S3 API and web endpoint on the host's address + ansible.builtin.uri: + url: "http://{{ ansible_default_ipv4.address }}:{{ item.port }}/" + headers: + Host: oats-runs + status_code: "{{ item.status }}" + loop: + - { port: 3900, status: 403 } + - { port: 3902, status: 404 } + + - name: Check RPC and the admin API stay on loopback + ansible.builtin.uri: + url: "http://{{ ansible_default_ipv4.address }}:{{ item }}/" + status_code: -1 + loop: [3901, 3903] diff --git a/storage/deploy/roles/oats_storage/tasks/access.yml b/storage/deploy/roles/oats_storage/tasks/access.yml deleted file mode 100644 index c32721d..0000000 --- a/storage/deploy/roles/oats_storage/tasks/access.yml +++ /dev/null @@ -1,33 +0,0 @@ ---- -- name: Create the directory mc is installed into - ansible.builtin.file: - path: "{{ oats_mc_path | dirname }}" - state: directory - mode: "0755" - -- name: Install the mc S3 client - ansible.builtin.get_url: - url: "{{ oats_mc_url }}" - dest: "{{ oats_mc_path }}" - mode: "0755" - -- name: Register the bucket with mc - ansible.builtin.command: >- - {{ oats_mc_path }} --config-dir {{ oats_conf_dir }}/mc alias set oats - {{ oats_s3_endpoint }} {{ oats_key_id }} {{ oats_key_secret }} - changed_when: false - no_log: true - -- name: Read the current CORS configuration - ansible.builtin.command: >- - {{ oats_mc_path }} --config-dir {{ oats_conf_dir }}/mc - cors get oats/{{ oats_bucket }} - register: oats_cors_now - changed_when: false - failed_when: false - -- name: Allow cross-origin reads - ansible.builtin.command: >- - {{ oats_mc_path }} --config-dir {{ oats_conf_dir }}/mc cors set oats/{{ oats_bucket }} - {{ oats_storage_dir }}/cors.xml - when: "'AllowedOrigin' not in oats_cors_now.stdout" diff --git a/storage/deploy/roles/oats_storage/tasks/bucket.yml b/storage/deploy/roles/oats_storage/tasks/bucket.yml new file mode 100644 index 0000000..5187e08 --- /dev/null +++ b/storage/deploy/roles/oats_storage/tasks/bucket.yml @@ -0,0 +1,29 @@ +--- +- name: Wait for Garage to accept commands + ansible.builtin.command: garage status + register: oats_status + until: oats_status.rc == 0 + retries: 30 + delay: 1 + changed_when: false + +- name: Read the bucket's state + ansible.builtin.shell: >- + curl -sS --fail -H "Authorization: Bearer $(cat {{ oats_conf_dir }}/admin_token)" + "{{ oats_admin_endpoint }}/v2/GetBucketInfo?globalAlias={{ oats_bucket }}" + register: oats_bucket_info + changed_when: false + +- name: Parse the bucket's state + ansible.builtin.set_fact: + oats_bucket_state: "{{ oats_bucket_info.stdout | from_json }}" + +- name: Open the bucket for anonymous reads + ansible.builtin.shell: >- + curl -sS --fail -H "Authorization: Bearer $(cat {{ oats_conf_dir }}/admin_token)" + -H "Content-Type: application/json" + -X POST "{{ oats_admin_endpoint }}/v2/UpdateBucket?id={{ oats_bucket_state.id }}" + --data {{ oats_bucket_public | to_json | quote }} + when: >- + not oats_bucket_state.websiteAccess + or oats_bucket_state.corsRules != oats_bucket_public.corsRules diff --git a/storage/deploy/roles/oats_storage/tasks/cluster.yml b/storage/deploy/roles/oats_storage/tasks/cluster.yml deleted file mode 100644 index dcd3663..0000000 --- a/storage/deploy/roles/oats_storage/tasks/cluster.yml +++ /dev/null @@ -1,62 +0,0 @@ ---- -- name: Wait for Garage to accept commands - ansible.builtin.command: "{{ oats_garage }} status" - args: - chdir: "{{ oats_storage_dir }}" - register: oats_status - until: oats_status.rc == 0 - retries: 30 - delay: 1 - changed_when: false - -- name: Give the node a layout role - when: "'NO ROLE ASSIGNED' in oats_status.stdout" - block: - - name: Stage the node's role - ansible.builtin.command: >- - {{ oats_garage }} layout assign -z {{ oats_zone }} -c {{ oats_capacity }} - {{ (oats_status.stdout_lines | select('search', 'NO ROLE ASSIGNED') | first).split() | first }} - args: - chdir: "{{ oats_storage_dir }}" - - - name: Read the staged layout version - ansible.builtin.command: "{{ oats_garage }} layout show" - args: - chdir: "{{ oats_storage_dir }}" - register: oats_layout - changed_when: false - - - name: Apply the staged layout - ansible.builtin.command: >- - {{ oats_garage }} layout apply --version - {{ (oats_layout.stdout | regex_search('layout version: (\d+)', '\1') | first | int) + 1 }} - args: - chdir: "{{ oats_storage_dir }}" - -- name: Look for the bucket - ansible.builtin.command: "{{ oats_garage }} bucket info {{ oats_bucket }}" - args: - chdir: "{{ oats_storage_dir }}" - register: oats_bucket_check - changed_when: false - failed_when: false - -- name: Create the bucket - ansible.builtin.command: "{{ oats_garage }} bucket create {{ oats_bucket }}" - args: - chdir: "{{ oats_storage_dir }}" - when: oats_bucket_check.rc != 0 - -- name: Read the bucket's state - ansible.builtin.command: "{{ oats_garage }} bucket info {{ oats_bucket }}" - args: - chdir: "{{ oats_storage_dir }}" - register: oats_bucket_state - changed_when: false - -# Anonymous read is served by the web endpoint, enabled per bucket. -- name: Serve the bucket's objects anonymously - ansible.builtin.command: "{{ oats_garage }} bucket website --allow {{ oats_bucket }}" - args: - chdir: "{{ oats_storage_dir }}" - when: "'Website access: true' not in oats_bucket_state.stdout" diff --git a/storage/deploy/roles/oats_storage/tasks/credentials.yml b/storage/deploy/roles/oats_storage/tasks/credentials.yml deleted file mode 100644 index db92fc5..0000000 --- a/storage/deploy/roles/oats_storage/tasks/credentials.yml +++ /dev/null @@ -1,57 +0,0 @@ ---- -- name: Look for the writer key - ansible.builtin.command: "{{ oats_garage }} key info {{ oats_key_name }}" - args: - chdir: "{{ oats_storage_dir }}" - register: oats_key_check - changed_when: false - failed_when: false - no_log: true - -- name: Create the writer key - ansible.builtin.command: "{{ oats_garage }} key create {{ oats_key_name }}" - args: - chdir: "{{ oats_storage_dir }}" - when: oats_key_check.rc != 0 - no_log: true - -- name: Grant the writer key access to the bucket - ansible.builtin.command: >- - {{ oats_garage }} bucket allow --read --write --owner - {{ oats_bucket }} --key {{ oats_key_name }} - args: - chdir: "{{ oats_storage_dir }}" - when: oats_bucket_state.stdout is not search('RWO\s+\S+\s+' ~ oats_key_name) - -# Garage holds the secret and will show it on demand, so the key is read back -# from the cluster rather than captured at creation. -- name: Read the writer key - ansible.builtin.command: "{{ oats_garage }} key info --show-secret {{ oats_key_name }}" - args: - chdir: "{{ oats_storage_dir }}" - register: oats_key_info - changed_when: false - no_log: true - -- name: Load the writer credentials - ansible.builtin.set_fact: - oats_key_id: "{{ oats_key_info.stdout | regex_search('Key ID: (\\S+)', '\\1') | first }}" - oats_key_secret: "{{ oats_key_info.stdout | regex_search('Secret key: (\\S+)', '\\1') | first }}" - no_log: true - -# A cache for anyone publishing by hand on the host; rewritten if it is lost. -- name: Store the writer credentials - ansible.builtin.copy: - content: | - AWS_ACCESS_KEY_ID={{ oats_key_id }} - AWS_SECRET_ACCESS_KEY={{ oats_key_secret }} - dest: "{{ oats_writer_env }}" - mode: "0600" - no_log: true - -- name: Copy the writer credentials to the control node - ansible.builtin.fetch: - src: "{{ oats_writer_env }}" - dest: "{{ oats_fetch_dest }}" - flat: true - when: oats_fetch_writer_env | bool diff --git a/storage/deploy/roles/oats_storage/tasks/disk.yml b/storage/deploy/roles/oats_storage/tasks/disk.yml deleted file mode 100644 index dd0e00f..0000000 --- a/storage/deploy/roles/oats_storage/tasks/disk.yml +++ /dev/null @@ -1,13 +0,0 @@ ---- -- name: Create a filesystem on the data device - community.general.filesystem: - fstype: "{{ oats_data_fstype }}" - dev: "{{ oats_data_device }}" - -- name: Mount the data device - ansible.posix.mount: - path: "{{ oats_data_mount }}" - src: "{{ oats_data_device }}" - fstype: "{{ oats_data_fstype }}" - opts: defaults,nofail - state: mounted diff --git a/storage/deploy/roles/oats_storage/tasks/main.yml b/storage/deploy/roles/oats_storage/tasks/main.yml index 9542c90..7b7e4de 100644 --- a/storage/deploy/roles/oats_storage/tasks/main.yml +++ b/storage/deploy/roles/oats_storage/tasks/main.yml @@ -1,26 +1,20 @@ --- -- name: Check that Docker is installed - ansible.builtin.command: docker compose version - changed_when: false - register: oats_docker_check - failed_when: oats_docker_check.rc != 0 +- name: Deploy the store + environment: + PATH: "{{ oats_bin_dir }}:{{ ansible_env.PATH }}" + RUST_LOG: error + block: + - name: Install and start the Garage service + ansible.builtin.import_tasks: service.yml -- name: Prepare the data disk - ansible.builtin.import_tasks: disk.yml - when: oats_data_device | length > 0 + - name: Open the bucket for anonymous reads + ansible.builtin.import_tasks: bucket.yml -- name: Deploy and start the Garage service - ansible.builtin.import_tasks: service.yml + - name: Publish the bucket under its hostname + ansible.builtin.import_tasks: proxy.yml + when: oats_public_hostname | length > 0 -- name: Set up the cluster and its bucket - ansible.builtin.import_tasks: cluster.yml - -- name: Issue the writer key - ansible.builtin.import_tasks: credentials.yml - -- name: Open the bucket for browser reads - ansible.builtin.import_tasks: access.yml - -- name: Publish the bucket under its hostname - ansible.builtin.import_tasks: proxy.yml - when: oats_public_hostname | length > 0 + - name: Show the new writer key, for the CI secrets + ansible.builtin.debug: + msg: "{{ oats_writer_key.stdout_lines | select('match', 'AWS_') }}" + when: oats_writer_env_file.changed diff --git a/storage/deploy/roles/oats_storage/tasks/proxy.yml b/storage/deploy/roles/oats_storage/tasks/proxy.yml index 8a6d8da..9cd2a2f 100644 --- a/storage/deploy/roles/oats_storage/tasks/proxy.yml +++ b/storage/deploy/roles/oats_storage/tasks/proxy.yml @@ -2,17 +2,13 @@ # Garage resolves the bucket from the request Host, so the hostname must # exist as an alias. - name: Read the bucket's aliases - ansible.builtin.command: "{{ oats_garage }} bucket info {{ oats_bucket }}" - args: - chdir: "{{ oats_storage_dir }}" + ansible.builtin.command: garage bucket info {{ oats_bucket }} register: oats_bucket_info changed_when: false - name: Alias the bucket to the public hostname ansible.builtin.command: >- - {{ oats_garage }} bucket alias {{ oats_bucket }} {{ oats_public_hostname }} - args: - chdir: "{{ oats_storage_dir }}" + garage bucket alias {{ oats_bucket }} {{ oats_public_hostname }} when: oats_public_hostname not in oats_bucket_info.stdout - name: Write the nginx vhost diff --git a/storage/deploy/roles/oats_storage/tasks/service.yml b/storage/deploy/roles/oats_storage/tasks/service.yml index 8700649..74096a4 100644 --- a/storage/deploy/roles/oats_storage/tasks/service.yml +++ b/storage/deploy/roles/oats_storage/tasks/service.yml @@ -1,62 +1,100 @@ --- +- name: Install the tools the role runs + ansible.builtin.apt: + name: [curl, openssl] + update_cache: true + cache_valid_time: 3600 + +- name: Create the service user + ansible.builtin.user: + name: "{{ oats_service_user }}" + system: true + shell: /usr/sbin/nologin + create_home: false + - name: Create the store directories ansible.builtin.file: path: "{{ item.path }}" state: directory mode: "{{ item.mode }}" + owner: "{{ item.owner | default(omit) }}" loop: + - { path: "{{ oats_bin_dir }}", mode: "0755" } - { path: "{{ oats_storage_dir }}", mode: "0755" } - - { path: "{{ oats_data_root }}/meta", mode: "0700" } - - { path: "{{ oats_data_root }}/data", mode: "0700" } - - { path: "{{ oats_conf_dir }}", mode: "0700" } - - { path: "{{ oats_conf_dir }}/mc", mode: "0700" } + - { path: "{{ oats_conf_dir }}", mode: "0755" } + - { path: "{{ oats_data_root }}/meta", mode: "0700", owner: "{{ oats_service_user }}" } + - { path: "{{ oats_data_root }}/data", mode: "0700", owner: "{{ oats_service_user }}" } -- name: Deploy the store configuration - ansible.builtin.copy: - src: "{{ playbook_dir }}/../{{ item.name }}" - dest: "{{ oats_storage_dir }}/{{ item.name }}" - mode: "{{ item.mode }}" - loop: - - { name: compose.yaml, mode: "0644" } - - { name: garage.toml, mode: "0644" } - - { name: cors.xml, mode: "0644" } - - { name: publish_run.sh, mode: "0755" } +- name: Install the garage binary + ansible.builtin.get_url: + url: "{{ oats_garage_url }}" + checksum: "sha256:{{ oats_garage_sha256 }}" + dest: "{{ oats_bin_dir }}/garage" + mode: "0755" notify: Restart Garage -- name: Point Garage's volumes at the data disk - ansible.builtin.template: - src: compose.override.yaml.j2 - dest: "{{ oats_storage_dir }}/compose.override.yaml" - mode: "0644" - notify: Restart Garage +- name: Deploy the publish script + ansible.builtin.copy: + src: "{{ role_path }}/../../../publish_run.sh" + dest: "{{ oats_storage_dir }}/publish_run.sh" + mode: "0755" # Generated once; a re-run never rotates them. -- name: Look for existing Garage secrets - ansible.builtin.stat: - path: "{{ oats_storage_dir }}/.env" - register: oats_env_file - -- name: Generate Garage secrets - ansible.builtin.set_fact: - oats_rpc_secret: "{{ lookup('ansible.builtin.password', '/dev/null', chars=['hexdigits'], length=64) | lower }}" - oats_admin_token: "{{ lookup('ansible.builtin.password', '/dev/null', length=32) }}" - when: not oats_env_file.stat.exists +- name: Generate the Garage secrets + ansible.builtin.copy: + content: "{{ lookup('ansible.builtin.password', '/dev/null', chars=['hexdigits'], length=64) | lower }}" + dest: "{{ oats_conf_dir }}/{{ item }}" + mode: "0600" + owner: "{{ oats_service_user }}" + force: false + loop: [rpc_secret, admin_token] no_log: true -- name: Write the Garage secrets file +- name: Generate the writer key + ansible.builtin.shell: | + set -e + id="GK$(openssl rand -hex 12)" + secret="$(openssl rand -hex 32)" + cat < Date: Tue, 8 Sep 2026 10:54:16 +0100 Subject: [PATCH 08/10] Tidy up instructions and improve legibility --- storage/README.md | 12 ++-- storage/publish_run.sh | 160 +++++++++++++++++++++-------------------- 2 files changed, 90 insertions(+), 82 deletions(-) diff --git a/storage/README.md b/storage/README.md index 47f6d31..0a02a49 100644 --- a/storage/README.md +++ b/storage/README.md @@ -76,9 +76,9 @@ server { } ``` -Set `oats_public_hostname` to that `server_name`: the role aliases the bucket to -it, which is how Garage maps the domain to the bucket. Set `oats_nginx_manage` -only when nginx runs on the store VM itself. +Set `oats_public_hostname` in the inventory to the `server_name` in nginx: the +role aliases the bucket to it, which is how Garage maps the domain to the +bucket. Set `oats_nginx_manage` only when nginx runs on the store VM itself. Garage starts with `--single-node --default-bucket`, which assigns the layout, creates the bucket and imports the writer key from `/etc/oats/writer.env`. The @@ -86,8 +86,7 @@ role generates that key on the first run and prints it at the end, for the CI secret store. Afterwards, `sudo cat /etc/oats/writer.env` on the host. Garage runs as the `garage` user under systemd, configured by -`/etc/garage.toml`; `garage status` on the host shows the node. To upgrade, snapshot the guest -(`qm snapshot pre-upgrade`), +`/etc/garage.toml`. `garage status` on the host shows the node. To update versions, set `oats_garage_version` and `oats_garage_sha256` to the new release and re-run the playbook. @@ -100,7 +99,8 @@ nothing changes, then checks the bucket is served anonymously with CORS: ```sh uv tool install molecule --with 'molecule-plugins[docker]' --with ansible-core -~/.local/share/uv/tools/molecule/bin/ansible-galaxy collection install community.docker +uv tool install ansible-core +ansible-galaxy collection install community.docker cd deploy/roles/oats_storage molecule test ``` diff --git a/storage/publish_run.sh b/storage/publish_run.sh index d709d4b..d6a86a9 100755 --- a/storage/publish_run.sh +++ b/storage/publish_run.sh @@ -28,61 +28,15 @@ usage() { exit "${1:-0}" } -RUN_DIR="" -TARGET="" -ENDPOINT="" -REINDEX=NO -MC="${OATS_MC:-mc}" - -while [ $# -gt 0 ]; do - case "$1" in - --endpoint) ENDPOINT="$2"; shift 2;; - --reindex) REINDEX=YES; shift;; - --mc) MC="$2"; shift 2;; - -h|--help) usage 0;; - -*) echo "Unknown option: $1" >&2; usage 1;; - *) - if [ -z "$RUN_DIR" ]; then RUN_DIR="$1" - elif [ -z "$TARGET" ]; then TARGET="$1" - else echo "Unexpected argument: $1" >&2; usage 1 - fi - shift;; - esac -done - -# --reindex takes no run directory, so the sole positional is the target -if [ "$REINDEX" = YES ] && [ -z "$TARGET" ]; then - TARGET="$RUN_DIR" - RUN_DIR="" -fi +die() { + echo "$*" >&2 + exit 1 +} -[ -n "$TARGET" ] || usage 1 -[ "$REINDEX" = YES ] || [ -n "$RUN_DIR" ] || usage 1 -case "$TARGET" in - s3://*) ;; - *) echo "Target must be an s3:// URL (e.g. s3://oats-runs)" >&2; usage 1;; -esac -[ -n "$ENDPOINT" ] || { echo "--endpoint is required" >&2; usage 1; } -[ -n "${AWS_ACCESS_KEY_ID:-}" ] && [ -n "${AWS_SECRET_ACCESS_KEY:-}" ] \ - || { echo "AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY must be set" >&2; exit 1; } - -command -v jq >/dev/null || { echo "jq is required" >&2; exit 1; } -command -v "$MC" >/dev/null || { echo "mc client not found: $MC" >&2; exit 1; } - -RUN_KEY="" -if [ "$REINDEX" = NO ]; then - MANIFEST="$RUN_DIR/run_manifest.json" - [ -f "$MANIFEST" ] || { echo "No run_manifest.json in $RUN_DIR" >&2; exit 1; } - - # The run key is the run's identity path, written by the harness. Fall back - # to the run directory's own trailing // path if absent. - RUN_KEY=$(jq -r '.run_key // empty' "$MANIFEST") - if [ -z "$RUN_KEY" ]; then - abs=$(cd "$RUN_DIR" && pwd) - RUN_KEY=$(echo "$abs" | rev | cut -d/ -f1-3 | rev) - fi - [ -n "$RUN_KEY" ] || { echo "Could not determine run key" >&2; exit 1; } -fi +# mc with the throwaway config dir created below. +s3() { + "$MC" --config-dir "$CFG" "$@" +} # jq program turning a stream of run manifests into the store index. Each run # contributes its identity plus an aggregate pass/fail count derived from the @@ -102,49 +56,103 @@ INDEX_JQ='{ } ] | sort_by(.timestamp) | reverse }' -# bucket[/prefix] from s3://bucket[/prefix] -path="${TARGET#s3://}" -alias="oats-publish-$$" -base="$alias/$path" +# The run key is the run's identity path, written by the harness. Fall back +# to the run directory's own trailing // path if absent. +run_key_of() { + local run_dir="$1" manifest="$1/run_manifest.json" key + [ -f "$manifest" ] || die "No run_manifest.json in $run_dir" -cfg=$(mktemp -d) -trap 'rm -rf "$cfg"' EXIT -"$MC" --config-dir "$cfg" alias set "$alias" "$ENDPOINT" \ - "$AWS_ACCESS_KEY_ID" "$AWS_SECRET_ACCESS_KEY" >/dev/null + key=$(jq -r '.run_key // empty' "$manifest") + [ -n "$key" ] || key=$(cd "$run_dir" && pwd | awk -F/ '{ print $(NF-2) "/" $(NF-1) "/" $NF }') + [ -n "$key" ] || die "Could not determine run key" + + echo "$key" +} # The manifests in the store are the source of truth and index.json is derived # from them, so it can be rebuilt at any time. It is assembled locally and # checked before upload: a failed enumeration must not replace a populated -# index with a short or empty one. +# index with a short or empty one. Prints the number of runs indexed. reindex() { - local required="${1:-}" keys="$cfg/keys" manifests="$cfg/manifests" index="$cfg/index.json" + local required="${1:-}" keys="$CFG/keys" manifests="$CFG/manifests" index="$CFG/index.json" count - "$MC" --config-dir "$cfg" find "$base" --name run_manifest.json > "$keys" + s3 find "$BASE" --name run_manifest.json > "$keys" : > "$manifests" while IFS= read -r key; do - "$MC" --config-dir "$cfg" cat "$key" >> "$manifests" + s3 cat "$key" >> "$manifests" done < "$keys" jq -s "$INDEX_JQ" < "$manifests" > "$index" - INDEX_COUNT=$(jq '.runs | length' "$index") + count=$(jq '.runs | length' "$index") - [ "$INDEX_COUNT" -gt 0 ] || { echo "Enumerated no runs; leaving index.json alone" >&2; exit 1; } + [ "$count" -gt 0 ] || die "Enumerated no runs; leaving index.json alone" if [ -n "$required" ]; then jq -e --arg k "$required" 'any(.runs[]; .key == $k)' "$index" >/dev/null \ - || { echo "$required missing from the rebuilt index; leaving index.json alone" >&2; exit 1; } + || die "$required missing from the rebuilt index; leaving index.json alone" fi - "$MC" --config-dir "$cfg" pipe "$base/index.json" < "$index" >/dev/null + s3 pipe "$BASE/index.json" < "$index" >/dev/null + echo "$count" } -if [ "$REINDEX" = YES ]; then - reindex - echo "Rebuilt $TARGET/index.json ($INDEX_COUNT runs)" +# --- Arguments --------------------------------------------------------------- + +ENDPOINT="" +REINDEX=false +MC="${OATS_MC:-mc}" +positional=() + +while [ $# -gt 0 ]; do + case "$1" in + --endpoint) ENDPOINT="$2"; shift 2;; + --reindex) REINDEX=true; shift;; + --mc) MC="$2"; shift 2;; + -h|--help) usage 0;; + -*) echo "Unknown option: $1" >&2; usage 1;; + *) positional+=("$1"); shift;; + esac +done + +RUN_DIR="" +if $REINDEX; then + [ ${#positional[@]} -eq 1 ] || usage 1 + TARGET="${positional[0]}" +else + [ ${#positional[@]} -eq 2 ] || usage 1 + RUN_DIR="${positional[0]}" + TARGET="${positional[1]}" +fi + +case "$TARGET" in + s3://*) ;; + *) echo "Target must be an s3:// URL (e.g. s3://oats-runs)" >&2; usage 1;; +esac +[ -n "$ENDPOINT" ] || { echo "--endpoint is required" >&2; usage 1; } +[ -n "${AWS_ACCESS_KEY_ID:-}" ] && [ -n "${AWS_SECRET_ACCESS_KEY:-}" ] \ + || die "AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY must be set" + +command -v jq >/dev/null || die "jq is required" +command -v "$MC" >/dev/null || die "mc client not found: $MC" + +# --- Publish ----------------------------------------------------------------- + +CFG=$(mktemp -d) +trap 'rm -rf "$CFG"' EXIT + +# /bucket[/prefix] from s3://bucket[/prefix] +ALIAS="oats-publish-$$" +BASE="$ALIAS/${TARGET#s3://}" +s3 alias set "$ALIAS" "$ENDPOINT" "$AWS_ACCESS_KEY_ID" "$AWS_SECRET_ACCESS_KEY" >/dev/null + +if $REINDEX; then + count=$(reindex) + echo "Rebuilt $TARGET/index.json ($count runs)" exit 0 fi -"$MC" --config-dir "$cfg" mirror --overwrite --remove "$RUN_DIR/" "$base/$RUN_KEY" >/dev/null -reindex "$RUN_KEY" +RUN_KEY=$(run_key_of "$RUN_DIR") +s3 mirror --overwrite --remove "$RUN_DIR/" "$BASE/$RUN_KEY" >/dev/null +count=$(reindex "$RUN_KEY") echo "Published run $RUN_KEY to $TARGET/$RUN_KEY" -echo "Regenerated $TARGET/index.json ($INDEX_COUNT runs)" +echo "Regenerated $TARGET/index.json ($count runs)" From 3140a1a8de3c4cf56e6e65cfb345f49fed21639a Mon Sep 17 00:00:00 2001 From: Michael Johnson Date: Tue, 8 Sep 2026 11:17:24 +0100 Subject: [PATCH 09/10] Clean up wording and add proxmox instructions --- storage/README.md | 33 ++++++++++++++++----------------- 1 file changed, 16 insertions(+), 17 deletions(-) diff --git a/storage/README.md b/storage/README.md index 0a02a49..8df3a5a 100644 --- a/storage/README.md +++ b/storage/README.md @@ -50,23 +50,15 @@ target needs systemd and ssh; the role installs the rest: - `bucket.yml` — anonymous read, CORS - `proxy.yml` — hostname alias, nginx vhost -```sh -cd deploy -cp inventory.example.yml inventory.yml # update host, user, hostname -ansible-playbook storage.yml -``` - -Data lives in `/var/lib/garage` by default but can be changed in -`inventory.yml`. The `oats_public_hostname` must match the nginx vhost. +Step one is to set up a separate VM in proxmox that will host our data. It needs +Debian or Ubuntu (apt), an ssh user with sudo, and the disk mounted at +`/var/lib/garage` (you can change the path in the inventory if preferred). -The S3 API listens on port 3900 and the web endpoint on 3902, on every -interface; RPC and the admin API stay on loopback. A reverse proxy on another -machine serves the store with: +Add a vhost to the `nginx` proxy: -eg. ```nginx server { - server_name oats-store.opendronemap.org; + server_name oats-store.opendronemap.org; # eg. location / { proxy_pass http://:3902; proxy_set_header Host $host; # Garage picks the bucket by Host @@ -76,14 +68,21 @@ server { } ``` +The S3 API listens on port 3900 and the web endpoint on 3902, on every +interface; RPC and the admin API stay on loopback. + Set `oats_public_hostname` in the inventory to the `server_name` in nginx: the role aliases the bucket to it, which is how Garage maps the domain to the bucket. Set `oats_nginx_manage` only when nginx runs on the store VM itself. -Garage starts with `--single-node --default-bucket`, which assigns the layout, -creates the bucket and imports the writer key from `/etc/oats/writer.env`. The -role generates that key on the first run and prints it at the end, for the CI -secret store. Afterwards, `sudo cat /etc/oats/writer.env` on the host. +```sh +cd deploy +cp inventory.example.yml inventory.yml # update host, user, hostname +ansible-playbook storage.yml +``` + +The role generates the writer key on the first run and prints it at the end, for +the CI secret store. Afterwards, `sudo cat /etc/oats/writer.env` on the host. Garage runs as the `garage` user under systemd, configured by `/etc/garage.toml`. `garage status` on the host shows the node. From 8ac323588d3a9848d8168ec452156ea2d70f035d Mon Sep 17 00:00:00 2001 From: Michael Johnson Date: Tue, 8 Sep 2026 11:34:04 +0100 Subject: [PATCH 10/10] Switch to rclone so we have an allowlist `mc` was easy, but only had a denylist. When we upload, it will be much easier to start with what we want and add, rather than risk huge files being uploaded. --- docs/ci.md | 9 +++--- storage/README.md | 17 +++++++++-- storage/publish_run.sh | 65 ++++++++++++++++++++++++++++-------------- 3 files changed, 62 insertions(+), 29 deletions(-) diff --git a/docs/ci.md b/docs/ci.md index d5d9998..cdbedf1 100644 --- a/docs/ci.md +++ b/docs/ci.md @@ -10,7 +10,7 @@ outside it, namely the runner, the cross-repo token and the ODM-side trigger. Register a runner against this repository (*Settings → Actions → Runners*). The default `self-hosted` label is all the workflow asks for. -It needs docker, plus `git`, `wget`, `rsync`, `sed`, `unzip`, `jq`, `mc`, and +It needs docker, plus `git`, `wget`, `rsync`, `sed`, `unzip`, `jq`, `rclone`, and `gdalinfo` on `PATH`; `./run` bootstraps bats itself. Give it plenty of RAM — the suite is RAM-bound and the `all` group sets the ceiling. Each full run generates about 60GB of output right now so as a rough guide we should aim for @@ -68,8 +68,9 @@ fails and the ODM publish job goes red. Every run uploads its reports and small text outputs as a GitHub artifact, kept for 14 days, which is enough to review a failure from the run page. -The whole run, ODM outputs included, goes to the artifact store described in -[`storage/README.md`](../storage/README.md) when the store is configured. The +The run's primary outputs go to the artifact store described in +[`storage/README.md`](../storage/README.md) when the store is configured; the +rest stays under `results/runs/` on the runner. The `Publish run to the store` step runs `storage/publish_run.sh`. To configure it, under *Settings → Secrets and variables → Actions* on this repository: @@ -78,4 +79,4 @@ under *Settings → Secrets and variables → Actions* on this repository: two values the storage playbook prints on its first run. The runner reaches the endpoint directly, so port 3900 on the store must be -open to it. The step needs `mc` and `jq` on the runner. +open to it. The step needs `rclone` and `jq` on the runner. diff --git a/storage/README.md b/storage/README.md index 8df3a5a..4774a05 100644 --- a/storage/README.md +++ b/storage/README.md @@ -8,17 +8,28 @@ the backend later if we need more storage etc. ## Store contract -Each run sits at its identity path, exactly as the harness writes it: +Each run sits at its identity path, laid out as the harness writes it but +holding only the primary outputs: ``` oats-runs (bucket root) index.json # every run, newest first /// # one self-contained run run_manifest.json + oats_manifest.tsv reports/*.xml - tests///... # the ODM outputs + tests/// + *.json, *.txt # log, options, task output + odm_orthophoto/odm_orthophoto.tif + odm_dem/*.tif + odm_georeferencing/odm_georeferenced_model.laz + odm_report/ ``` +Input images, OpenSfM/OpenMVS intermediates, meshes and textures stay on the +runner; the allowlist is `PUBLISH_FILTER` in `publish_run.sh`. A full run is +about 2GB in the store against 50-60GB on the runner. + Each run is stored under its `run_key` from `run_manifest.json`. `index.json` is all the viewer reads to find runs: @@ -113,7 +124,7 @@ molecule test ./publish_run.sh s3://oats-runs --endpoint http://:3900 ``` -Idempotent per run key. Needs `mc`, `jq`, and `AWS_ACCESS_KEY_ID` / +Idempotent per run key. Needs `rclone`, `jq`, and `AWS_ACCESS_KEY_ID` / `AWS_SECRET_ACCESS_KEY`. See `--help`. `index.json` is derived from the run manifests, so `--reindex` rebuilds it from diff --git a/storage/publish_run.sh b/storage/publish_run.sh index d6a86a9..2d85dc0 100755 --- a/storage/publish_run.sh +++ b/storage/publish_run.sh @@ -3,6 +3,11 @@ # Publish an OATS run directory to the Garage (S3) store and regenerate the # store's index.json. Idempotent per run key. See storage/README.md. # +# Only the primary outputs are published: the run manifests and reports, each +# test's logs and options, and per test the orthophoto, DEMs, georeferenced +# point cloud and report. Input images, OpenSfM/OpenMVS intermediates, meshes +# and textures stay on the runner. +# # Usage: # publish_run.sh --endpoint URL # publish_run.sh --reindex --endpoint URL @@ -15,11 +20,12 @@ # --endpoint URL S3 endpoint, e.g. http://127.0.0.1:3900 (required) # --reindex Rebuild index.json from the manifests already in the # store, without publishing a run -# --mc PATH mc binary to use (default: mc on PATH, or $OATS_MC) # -h, --help Show this help # # S3 credentials are read from the environment: # AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY +# +# Needs rclone and jq. set -euo pipefail @@ -33,11 +39,6 @@ die() { exit 1 } -# mc with the throwaway config dir created below. -s3() { - "$MC" --config-dir "$CFG" "$@" -} - # jq program turning a stream of run manifests into the store index. Each run # contributes its identity plus an aggregate pass/fail count derived from the # per-dataset bats exit codes. @@ -56,6 +57,22 @@ INDEX_JQ='{ } ] | sort_by(.timestamp) | reverse }' +# rclone filter selecting what is published. First match wins; the final rule +# drops everything not listed, and rclone does not descend into directories +# nothing could match, so opensfm/, images/ and submodels/ are never walked. +PUBLISH_FILTER=' ++ /run_manifest.json ++ /oats_manifest.tsv ++ /reports/** ++ /tests/*/*/*.json ++ /tests/*/*/*.txt ++ /tests/*/*/odm_orthophoto/odm_orthophoto.tif ++ /tests/*/*/odm_dem/*.tif ++ /tests/*/*/odm_georeferencing/odm_georeferenced_model.laz ++ /tests/*/*/odm_report/** +- * +' + # The run key is the run's identity path, written by the harness. Fall back # to the run directory's own trailing // path if absent. run_key_of() { @@ -74,14 +91,9 @@ run_key_of() { # checked before upload: a failed enumeration must not replace a populated # index with a short or empty one. Prints the number of runs indexed. reindex() { - local required="${1:-}" keys="$CFG/keys" manifests="$CFG/manifests" index="$CFG/index.json" count - - s3 find "$BASE" --name run_manifest.json > "$keys" - : > "$manifests" - while IFS= read -r key; do - s3 cat "$key" >> "$manifests" - done < "$keys" + local required="${1:-}" manifests="$CFG/manifests" index="$CFG/index.json" count + rclone cat "$STORE" --include 'run_manifest.json' > "$manifests" jq -s "$INDEX_JQ" < "$manifests" > "$index" count=$(jq '.runs | length' "$index") @@ -91,7 +103,7 @@ reindex() { || die "$required missing from the rebuilt index; leaving index.json alone" fi - s3 pipe "$BASE/index.json" < "$index" >/dev/null + rclone rcat "$STORE/index.json" < "$index" echo "$count" } @@ -99,14 +111,12 @@ reindex() { ENDPOINT="" REINDEX=false -MC="${OATS_MC:-mc}" positional=() while [ $# -gt 0 ]; do case "$1" in --endpoint) ENDPOINT="$2"; shift 2;; --reindex) REINDEX=true; shift;; - --mc) MC="$2"; shift 2;; -h|--help) usage 0;; -*) echo "Unknown option: $1" >&2; usage 1;; *) positional+=("$1"); shift;; @@ -132,17 +142,26 @@ esac || die "AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY must be set" command -v jq >/dev/null || die "jq is required" -command -v "$MC" >/dev/null || die "mc client not found: $MC" +command -v rclone >/dev/null || die "rclone is required" # --- Publish ----------------------------------------------------------------- CFG=$(mktemp -d) trap 'rm -rf "$CFG"' EXIT -# /bucket[/prefix] from s3://bucket[/prefix] -ALIAS="oats-publish-$$" -BASE="$ALIAS/${TARGET#s3://}" -s3 alias set "$ALIAS" "$ENDPOINT" "$AWS_ACCESS_KEY_ID" "$AWS_SECRET_ACCESS_KEY" >/dev/null +# An on-the-fly S3 remote against the Garage endpoint, credentials from the +# AWS_* variables. The empty config file stops rclone announcing that it has +# none; the key never appears on a command line. +: > "$CFG/rclone.conf" +export RCLONE_CONFIG="$CFG/rclone.conf" +export RCLONE_S3_PROVIDER=Other +export RCLONE_S3_ENDPOINT="$ENDPOINT" +export RCLONE_S3_REGION=garage +export RCLONE_S3_ENV_AUTH=true +export RCLONE_S3_NO_CHECK_BUCKET=true + +# :s3:bucket[/prefix] from s3://bucket[/prefix] +STORE=":s3:${TARGET#s3://}" if $REINDEX; then count=$(reindex) @@ -151,7 +170,9 @@ if $REINDEX; then fi RUN_KEY=$(run_key_of "$RUN_DIR") -s3 mirror --overwrite --remove "$RUN_DIR/" "$BASE/$RUN_KEY" >/dev/null +printf '%s' "$PUBLISH_FILTER" > "$CFG/filter" +# --delete-excluded also trims anything a fuller earlier publish left behind. +rclone sync "$RUN_DIR" "$STORE/$RUN_KEY" --filter-from "$CFG/filter" --delete-excluded count=$(reindex "$RUN_KEY") echo "Published run $RUN_KEY to $TARGET/$RUN_KEY"