From 08d769c17b24ca658fca9b73cbd9ef3a91e8c32c Mon Sep 17 00:00:00 2001 From: Michael Johnson Date: Wed, 15 Jul 2026 19:24:20 +0100 Subject: [PATCH 1/2] Add self-hosted CI workflow for the OATS suite Runs ./run against an ODM docker image on a self-hosted runner and publishes the JUnit reports to the Checks tab and the run directory as a build artifact. Triggers on workflow_dispatch (image/tag/group), push and pull_request on master (fast small group, fork PRs skipped), and a repository_dispatch odm-image-published event so ODM can kick a run after publishing an image. Runs are serialized with a concurrency group. The runner is CPU-only, so the suite covers ODM's CPU path and the GPU image is not tested. docs/ci.md covers the runner setup, the repository_dispatch payload contract, the ODM-side dispatch step and the OATS_DISPATCH_TOKEN secret. --- .github/workflows/oats.yml | 161 +++++++++++++++++++++++++++++++++++++ docs/ci.md | 73 +++++++++++++++++ 2 files changed, 234 insertions(+) create mode 100644 .github/workflows/oats.yml create mode 100644 docs/ci.md diff --git a/.github/workflows/oats.yml b/.github/workflows/oats.yml new file mode 100644 index 0000000..f26b2a9 --- /dev/null +++ b/.github/workflows/oats.yml @@ -0,0 +1,161 @@ +# Runs the OATS suite against an ODM docker image on a self-hosted runner and +# publishes the JUnit reports and the run artifact. +# +# Triggers: +# * workflow_dispatch - manual run, choose image/tag/group. +# * push / pull_request - validate the harness itself on OATS master +# (fast `small` group). +# * repository_dispatch - fired by ODM after it publishes a new image. +# +# repository_dispatch contract (event_type: odm-image-published): +# { +# "event_type": "odm-image-published", +# "client_payload": { +# "image": "opendronemap/odm", # image repository (no tag) +# "tag": "latest" # tag that was just published +# } +# } +# +# Runner requirements and concurrency are documented in docs/ci.md. + +name: OATS + +on: + workflow_dispatch: + inputs: + image: + description: Docker image to test (repository without tag) + default: opendronemap/odm + tag: + description: Image tag to test + default: latest + group: + description: Dataset group to run + type: choice + default: all + options: + - all + - small + push: + branches: [master] + paths-ignore: + - '**.md' + - docs/** + - LICENSE + pull_request: + branches: [master] + paths-ignore: + - '**.md' + - docs/** + - LICENSE + repository_dispatch: + types: [odm-image-published] + +# Serialize: OATS runs are heavy and RAM-bound, so never let two overlap on a +# runner. Queued runs wait rather than cancel, so an in-flight run finishes. +concurrency: + group: oats-${{ github.repository }} + cancel-in-progress: false + +jobs: + oats: + # Skip pull requests coming from forks: they must not execute untrusted + # code on the self-hosted runner. Fork PRs can be validated with a manual + # workflow_dispatch after review. + if: >- + github.event_name != 'pull_request' || + github.event.pull_request.head.repo.full_name == github.repository + runs-on: self-hosted + timeout-minutes: 1440 + steps: + - name: Checkout OATS + uses: actions/checkout@v7 + + - name: Resolve run parameters + id: params + env: + EVENT_NAME: ${{ github.event_name }} + INPUT_IMAGE: ${{ github.event.inputs.image }} + INPUT_TAG: ${{ github.event.inputs.tag }} + INPUT_GROUP: ${{ github.event.inputs.group }} + DISPATCH_IMAGE: ${{ github.event.client_payload.image }} + DISPATCH_TAG: ${{ github.event.client_payload.tag }} + run: | + set -euo pipefail + case "$EVENT_NAME" in + workflow_dispatch) + image="${INPUT_IMAGE:-opendronemap/odm}" + tag="${INPUT_TAG:-latest}" + group="${INPUT_GROUP:-all}" + ;; + repository_dispatch) + image="${DISPATCH_IMAGE:-opendronemap/odm}" + tag="${DISPATCH_TAG:-latest}" + group="all" + ;; + *) + # push / pull_request validate the harness itself: keep it fast. + image="opendronemap/odm" + tag="latest" + group="small" + ;; + esac + { + echo "image=$image" + echo "tag=$tag" + echo "group=$group" + } >> "$GITHUB_OUTPUT" + echo "Resolved image=$image tag=$tag group=$group (event=$EVENT_NAME)" + + - name: Pull ODM image + run: docker pull "${{ steps.params.outputs.image }}:${{ steps.params.outputs.tag }}" + + - name: Clear previous run results + # The self-hosted workspace persists between runs; drop old run + # directories so the newest run_manifest.json is unambiguous. Downloaded + # datasets under datasets/ are left in place to avoid re-fetching them. + run: rm -rf results/runs + + - name: Run OATS + id: run + run: | + ./run "${{ steps.params.outputs.group }}" \ + --image_name "${{ steps.params.outputs.image }}" \ + --tag "${{ steps.params.outputs.tag }}" + + - name: Locate run directory + id: rundir + if: always() + run: | + set -euo pipefail + manifest=$(find results/runs -name run_manifest.json -printf '%T@\t%p\n' 2>/dev/null \ + | sort -rn | head -1 | cut -f2-) + if [ -z "$manifest" ]; then + echo "No run_manifest.json found under results/runs" >&2 + exit 1 + fi + run_dir=$(dirname "$manifest") + echo "run_dir=$run_dir" >> "$GITHUB_OUTPUT" + echo "Run directory: $run_dir" + + - name: Publish JUnit results + if: always() && steps.rundir.outputs.run_dir != '' + uses: mikepenz/action-junit-report@v6 + with: + report_paths: ${{ steps.rundir.outputs.run_dir }}/reports/*.xml + check_name: OATS results + require_tests: true + fail_on_failure: true + + # TODO(storage): once the self-hosted MinIO storage service exists + # (separate task), push $run_dir (including run_manifest.json) to the + # bucket, e.g. `mc cp --recursive "$run_dir" oats/`. Until then + # the run directory is exposed as a GitHub Actions artifact below. + - name: Upload run artifact + if: always() && steps.rundir.outputs.run_dir != '' + uses: actions/upload-artifact@v7 + with: + name: oats-run-${{ github.run_id }} + path: ${{ steps.rundir.outputs.run_dir }} + if-no-files-found: warn + retention-days: 14 diff --git a/docs/ci.md b/docs/ci.md new file mode 100644 index 0000000..bd2acab --- /dev/null +++ b/docs/ci.md @@ -0,0 +1,73 @@ +# CI + +`.github/workflows/oats.yml` runs the suite on a self-hosted runner: on demand, +when OATS master changes, and when ODM publishes a new image. The workflow +itself is the reference for what runs when — this covers the parts that live +outside it, namely the runner, the cross-repo token and the ODM-side trigger. + +## The runner + +Register a runner against this repository (*Settings → Actions → Runners*). The +default `self-hosted` label is all the workflow asks for. + +It needs docker, plus `git`, `wget`, `rsync`, `sed`, `unzip` and `jq` on `PATH`; +`./run` bootstraps bats itself. Give it plenty of RAM — the suite is RAM-bound +and the `all` group sets the ceiling — and tens of GB of disk. The workspace +persists between runs: datasets are cached under `datasets/`, and every run +writes a full set of ODM outputs under `results/runs/`. Nothing prunes those +yet. + +Keep a single runner attached. The workflow serialises runs with a `concurrency` +group, but that only helps if there is one machine to serialise onto — a second +runner would happily pick up a queued run alongside the first. + +## No GPU + +I assume the runner has no GPU, so the suite only exercises ODM's CPU path, and +`opendronemap/odm:gpu` is not tested at all — `publish-docker-gpu.yaml` has no +dispatch step by design. Testing it would need GPU hardware with +`nvidia-container-toolkit`, a `--gpus` flag in the harness's `docker run` (it +passes none today), and the dispatch step added on the ODM side. + +## The ODM trigger + +ODM's `publish-docker.yaml` dispatches to OATS after pushing a new image. The +payload contract is at the top of `oats.yml`; the ODM-side step is: + +```yaml + - name: Trigger OATS test suite + run: | + curl -sSf -X POST \ + -H "Accept: application/vnd.github+json" \ + -H "Authorization: Bearer ${{ secrets.OATS_DISPATCH_TOKEN }}" \ + https://api.github.com/repos/OpenDroneMap/oats/dispatches \ + -d '{"event_type":"odm-image-published","client_payload":{"image":"opendronemap/odm","tag":"latest"}}' +``` + +`OATS_DISPATCH_TOKEN` is a secret on the **ODM** repository, not this one. The +`GITHUB_TOKEN` Actions provides will not do: it is scoped to the repository +running the workflow, and events raised with it never start a workflow run. + +To generate one, from an account with admin on this repo: + +1. *Settings → Developer settings → Personal access tokens → Fine-grained + tokens → Generate new token*. +2. Resource owner **OpenDroneMap**. Org-owned tokens need an org owner to + approve the request before they work. +3. Repository access: *Only select repositories* → `OpenDroneMap/oats`. +4. Repository permissions: **Contents → Read and write**. Broader than we want, + but `repository_dispatch` has no permission of its own and GitHub files it + under Contents. +5. Add it to ODM under *Settings → Secrets and variables → Actions → New + repository secret*, named `OATS_DISPATCH_TOKEN`. + +Fine-grained tokens expire, so note the date. When one lapses the dispatch step +fails and the ODM publish job goes red — the image is already pushed by then, so +it is only the OATS run that is lost. A classic PAT with `repo` scope works too +and can be set never to expire, at the cost of reaching every repo its owner can. + +## Results + +Runs upload the run directory as a GitHub artifact with a 14-day retention. +That is a placeholder until the storage service lands — see the +`TODO(storage)` in `oats.yml`. From d7c193b1818e1a6436c027fd8021ef6ddff68db6 Mon Sep 17 00:00:00 2001 From: Michael Johnson Date: Wed, 5 Aug 2026 10:42:12 +0100 Subject: [PATCH 2/2] Don't store the full output The output is far too large to store as a github artifact. Cut to only the text outputs. --- .github/workflows/oats.yml | 20 +++++++++++++++----- docs/ci.md | 21 +++++++++------------ 2 files changed, 24 insertions(+), 17 deletions(-) diff --git a/.github/workflows/oats.yml b/.github/workflows/oats.yml index f26b2a9..b8ab885 100644 --- a/.github/workflows/oats.yml +++ b/.github/workflows/oats.yml @@ -147,15 +147,25 @@ jobs: require_tests: true fail_on_failure: true - # TODO(storage): once the self-hosted MinIO storage service exists - # (separate task), push $run_dir (including run_manifest.json) to the - # bucket, e.g. `mc cp --recursive "$run_dir" oats/`. Until then - # the run directory is exposed as a GitHub Actions artifact below. + # TODO: This will need updating once the self-hosted Garage storage + # service exists For now we just store the text artifacts as the whole + # results directory is far too big to store as an artifact. - name: Upload run artifact if: always() && steps.rundir.outputs.run_dir != '' uses: actions/upload-artifact@v7 with: name: oats-run-${{ github.run_id }} - path: ${{ steps.rundir.outputs.run_dir }} + path: | + ${{ steps.rundir.outputs.run_dir }}/run_manifest.json + ${{ steps.rundir.outputs.run_dir }}/oats_manifest.tsv + ${{ steps.rundir.outputs.run_dir }}/reports/*.xml + ${{ steps.rundir.outputs.run_dir }}/tests/*/*/task_output.txt + ${{ steps.rundir.outputs.run_dir }}/tests/*/*/log.json + ${{ steps.rundir.outputs.run_dir }}/tests/*/*/benchmark.txt + ${{ steps.rundir.outputs.run_dir }}/tests/*/*/options.json + ${{ steps.rundir.outputs.run_dir }}/tests/*/*/img_list.txt + ${{ steps.rundir.outputs.run_dir }}/tests/*/*/cameras.json + ${{ steps.rundir.outputs.run_dir }}/tests/*/*/odm_report/stats.json + ${{ steps.rundir.outputs.run_dir }}/tests/*/*/odm_report/shots.geojson if-no-files-found: warn retention-days: 14 diff --git a/docs/ci.md b/docs/ci.md index bd2acab..3f66db3 100644 --- a/docs/ci.md +++ b/docs/ci.md @@ -10,12 +10,11 @@ outside it, namely the runner, the cross-repo token and the ODM-side trigger. Register a runner against this repository (*Settings → Actions → Runners*). The default `self-hosted` label is all the workflow asks for. -It needs docker, plus `git`, `wget`, `rsync`, `sed`, `unzip` and `jq` on `PATH`; -`./run` bootstraps bats itself. Give it plenty of RAM — the suite is RAM-bound -and the `all` group sets the ceiling — and tens of GB of disk. The workspace -persists between runs: datasets are cached under `datasets/`, and every run -writes a full set of ODM outputs under `results/runs/`. Nothing prunes those -yet. +It needs docker, plus `git`, `wget`, `rsync`, `sed`, `unzip`, `jq`, and +`gdalinfo` on `PATH`; `./run` bootstraps bats itself. Give it plenty of RAM — +the suite is RAM-bound and the `all` group sets the ceiling. Each full run +generates about 60GB of output right now so as a rough guide we should aim for +about 1TB of disk space. Keep a single runner attached. The workflow serialises runs with a `concurrency` group, but that only helps if there is one machine to serialise onto — a second @@ -62,12 +61,10 @@ To generate one, from an account with admin on this repo: repository secret*, named `OATS_DISPATCH_TOKEN`. Fine-grained tokens expire, so note the date. When one lapses the dispatch step -fails and the ODM publish job goes red — the image is already pushed by then, so -it is only the OATS run that is lost. A classic PAT with `repo` scope works too -and can be set never to expire, at the cost of reaching every repo its owner can. +fails and the ODM publish job goes red. ## Results -Runs upload the run directory as a GitHub artifact with a 14-day retention. -That is a placeholder until the storage service lands — see the -`TODO(storage)` in `oats.yml`. +Currently we store the text outputs as a github artifact which will allow us to +review any failures. This will be replaced by the s3-compatible Garage server +once its set up.