First pass at CI workflow

.github/workflows/oats.yml

@@ -0,0 +1,171 @@
+# Runs the OATS suite against an ODM docker image on a self-hosted runner and
+# publishes the JUnit reports and the run artifact.
+#
+# Triggers:
+#   * workflow_dispatch     - manual run, choose image/tag/group.
+#   * push / pull_request   - validate the harness itself on OATS master
+#                             (fast `small` group).
+#   * repository_dispatch   - fired by ODM after it publishes a new image.
+#
+# repository_dispatch contract (event_type: odm-image-published):
+#   {
+#     "event_type": "odm-image-published",
+#     "client_payload": {
+#       "image": "opendronemap/odm",   # image repository (no tag)
+#       "tag":   "latest"              # tag that was just published
+#     }
+#   }
+#
+# Runner requirements and concurrency are documented in docs/ci.md.
+
+name: OATS
+
+on:
+  workflow_dispatch:
+    inputs:
+      image:
+        description: Docker image to test (repository without tag)
+        default: opendronemap/odm
+      tag:
+        description: Image tag to test
+        default: latest
+      group:
+        description: Dataset group to run
+        type: choice
+        default: all
+        options:
+          - all
+          - small
+  push:
+    branches: [master]
+    paths-ignore:
+      - '**.md'
+      - docs/**
+      - LICENSE
+  pull_request:
+    branches: [master]
+    paths-ignore:
+      - '**.md'
+      - docs/**
+      - LICENSE
+  repository_dispatch:
+    types: [odm-image-published]
+
+# Serialize: OATS runs are heavy and RAM-bound, so never let two overlap on a
+# runner. Queued runs wait rather than cancel, so an in-flight run finishes.
+concurrency:
+  group: oats-${{ github.repository }}
+  cancel-in-progress: false
+
+jobs:
+  oats:
+    # Skip pull requests coming from forks: they must not execute untrusted
+    # code on the self-hosted runner. Fork PRs can be validated with a manual
+    # workflow_dispatch after review.
+    if: >-
+      github.event_name != 'pull_request' ||
+      github.event.pull_request.head.repo.full_name == github.repository
+    runs-on: self-hosted
+    timeout-minutes: 1440
+    steps:
+      - name: Checkout OATS
+        uses: actions/checkout@v7
+
+      - name: Resolve run parameters
+        id: params
+        env:
+          EVENT_NAME: ${{ github.event_name }}
+          INPUT_IMAGE: ${{ github.event.inputs.image }}
+          INPUT_TAG: ${{ github.event.inputs.tag }}
+          INPUT_GROUP: ${{ github.event.inputs.group }}
+          DISPATCH_IMAGE: ${{ github.event.client_payload.image }}
+          DISPATCH_TAG: ${{ github.event.client_payload.tag }}
+        run: |
+          set -euo pipefail
+          case "$EVENT_NAME" in
+            workflow_dispatch)
+              image="${INPUT_IMAGE:-opendronemap/odm}"
+              tag="${INPUT_TAG:-latest}"
+              group="${INPUT_GROUP:-all}"
+              ;;
+            repository_dispatch)
+              image="${DISPATCH_IMAGE:-opendronemap/odm}"
+              tag="${DISPATCH_TAG:-latest}"
+              group="all"
+              ;;
+            *)
+              # push / pull_request validate the harness itself: keep it fast.
+              image="opendronemap/odm"
+              tag="latest"
+              group="small"
+              ;;
+          esac
+          {
+            echo "image=$image"
+            echo "tag=$tag"
+            echo "group=$group"
+          } >> "$GITHUB_OUTPUT"
+          echo "Resolved image=$image tag=$tag group=$group (event=$EVENT_NAME)"
+
+      - name: Pull ODM image
+        run: docker pull "${{ steps.params.outputs.image }}:${{ steps.params.outputs.tag }}"
+
+      - name: Clear previous run results
+        # The self-hosted workspace persists between runs; drop old run
+        # directories so the newest run_manifest.json is unambiguous. Downloaded
+        # datasets under datasets/ are left in place to avoid re-fetching them.
+        run: rm -rf results/runs
+
+      - name: Run OATS
+        id: run
+        run: |
+          ./run "${{ steps.params.outputs.group }}" \
+            --image_name "${{ steps.params.outputs.image }}" \
+            --tag "${{ steps.params.outputs.tag }}"
+
+      - name: Locate run directory
+        id: rundir
+        if: always()
+        run: |
+          set -euo pipefail
+          manifest=$(find results/runs -name run_manifest.json -printf '%T@\t%p\n' 2>/dev/null \
+            | sort -rn | head -1 | cut -f2-)
+          if [ -z "$manifest" ]; then
+            echo "No run_manifest.json found under results/runs" >&2
+            exit 1
+          fi
+          run_dir=$(dirname "$manifest")
+          echo "run_dir=$run_dir" >> "$GITHUB_OUTPUT"
+          echo "Run directory: $run_dir"
+
+      - name: Publish JUnit results
+        if: always() && steps.rundir.outputs.run_dir != ''
+        uses: mikepenz/action-junit-report@v6
+        with:
+          report_paths: ${{ steps.rundir.outputs.run_dir }}/reports/*.xml
+          check_name: OATS results
+          require_tests: true
+          fail_on_failure: true
+
+      # TODO: This will need updating once the self-hosted Garage storage
+      # service exists For now we just store the text artifacts as the whole
+      # results directory is far too big to store as an artifact.
+      - name: Upload run artifact
+        if: always() && steps.rundir.outputs.run_dir != ''
+        uses: actions/upload-artifact@v7
+        with:
+          name: oats-run-${{ github.run_id }}
+          path: |
+            ${{ steps.rundir.outputs.run_dir }}/run_manifest.json
+            ${{ steps.rundir.outputs.run_dir }}/oats_manifest.tsv
+            ${{ steps.rundir.outputs.run_dir }}/reports/*.xml
+            ${{ steps.rundir.outputs.run_dir }}/tests/*/*/task_output.txt
+            ${{ steps.rundir.outputs.run_dir }}/tests/*/*/log.json
+            ${{ steps.rundir.outputs.run_dir }}/tests/*/*/benchmark.txt
+            ${{ steps.rundir.outputs.run_dir }}/tests/*/*/options.json
+            ${{ steps.rundir.outputs.run_dir }}/tests/*/*/img_list.txt
+            ${{ steps.rundir.outputs.run_dir }}/tests/*/*/cameras.json
+            ${{ steps.rundir.outputs.run_dir }}/tests/*/*/odm_report/stats.json
+            ${{ steps.rundir.outputs.run_dir }}/tests/*/*/odm_report/shots.geojson
+          if-no-files-found: warn
+          retention-days: 14

docs/ci.md

@@ -0,0 +1,70 @@
+# CI
+
+`.github/workflows/oats.yml` runs the suite on a self-hosted runner: on demand,
+when OATS master changes, and when ODM publishes a new image. The workflow
+itself is the reference for what runs when — this covers the parts that live
+outside it, namely the runner, the cross-repo token and the ODM-side trigger.
+
+## The runner
+
+Register a runner against this repository (*Settings → Actions → Runners*). The
+default `self-hosted` label is all the workflow asks for.
+
+It needs docker, plus `git`, `wget`, `rsync`, `sed`, `unzip`, `jq`, and
+`gdalinfo` on `PATH`; `./run` bootstraps bats itself. Give it plenty of RAM —
+the suite is RAM-bound and the `all` group sets the ceiling. Each full run
+generates about 60GB of output right now so as a rough guide we should aim for
+about 1TB of disk space.
+
+Keep a single runner attached. The workflow serialises runs with a `concurrency`
+group, but that only helps if there is one machine to serialise onto — a second
+runner would happily pick up a queued run alongside the first.
+
+## No GPU
+
+I assume the runner has no GPU, so the suite only exercises ODM's CPU path, and
+`opendronemap/odm:gpu` is not tested at all — `publish-docker-gpu.yaml` has no
+dispatch step by design. Testing it would need GPU hardware with
+`nvidia-container-toolkit`, a `--gpus` flag in the harness's `docker run` (it
+passes none today), and the dispatch step added on the ODM side.
+
+## The ODM trigger
+
+ODM's `publish-docker.yaml` dispatches to OATS after pushing a new image. The
+payload contract is at the top of `oats.yml`; the ODM-side step is:
+
+```yaml
+    - name: Trigger OATS test suite
+      run: |
+        curl -sSf -X POST \
+          -H "Accept: application/vnd.github+json" \
+          -H "Authorization: Bearer ${{ secrets.OATS_DISPATCH_TOKEN }}" \
+          https://api.github.com/repos/OpenDroneMap/oats/dispatches \
+          -d '{"event_type":"odm-image-published","client_payload":{"image":"opendronemap/odm","tag":"latest"}}'
+```
+
+`OATS_DISPATCH_TOKEN` is a secret on the **ODM** repository, not this one. The
+`GITHUB_TOKEN` Actions provides will not do: it is scoped to the repository
+running the workflow, and events raised with it never start a workflow run.
+
+To generate one, from an account with admin on this repo:
+
+1. *Settings → Developer settings → Personal access tokens → Fine-grained
+   tokens → Generate new token*.
+2. Resource owner **OpenDroneMap**. Org-owned tokens need an org owner to
+   approve the request before they work.
+3. Repository access: *Only select repositories* → `OpenDroneMap/oats`.
+4. Repository permissions: **Contents → Read and write**. Broader than we want,
+   but `repository_dispatch` has no permission of its own and GitHub files it
+   under Contents.
+5. Add it to ODM under *Settings → Secrets and variables → Actions → New
+   repository secret*, named `OATS_DISPATCH_TOKEN`.
+
+Fine-grained tokens expire, so note the date. When one lapses the dispatch step
+fails and the ODM publish job goes red.
+
+## Results
+
+Currently we store the text outputs as a github artifact which will allow us to
+review any failures. This will be replaced by the s3-compatible Garage server
+once its set up.