diff --git a/.github/workflows/oats.yml b/.github/workflows/oats.yml new file mode 100644 index 0000000..b8ab885 --- /dev/null +++ b/.github/workflows/oats.yml @@ -0,0 +1,171 @@ +# Runs the OATS suite against an ODM docker image on a self-hosted runner and +# publishes the JUnit reports and the run artifact. +# +# Triggers: +# * workflow_dispatch - manual run, choose image/tag/group. +# * push / pull_request - validate the harness itself on OATS master +# (fast `small` group). +# * repository_dispatch - fired by ODM after it publishes a new image. +# +# repository_dispatch contract (event_type: odm-image-published): +# { +# "event_type": "odm-image-published", +# "client_payload": { +# "image": "opendronemap/odm", # image repository (no tag) +# "tag": "latest" # tag that was just published +# } +# } +# +# Runner requirements and concurrency are documented in docs/ci.md. + +name: OATS + +on: + workflow_dispatch: + inputs: + image: + description: Docker image to test (repository without tag) + default: opendronemap/odm + tag: + description: Image tag to test + default: latest + group: + description: Dataset group to run + type: choice + default: all + options: + - all + - small + push: + branches: [master] + paths-ignore: + - '**.md' + - docs/** + - LICENSE + pull_request: + branches: [master] + paths-ignore: + - '**.md' + - docs/** + - LICENSE + repository_dispatch: + types: [odm-image-published] + +# Serialize: OATS runs are heavy and RAM-bound, so never let two overlap on a +# runner. Queued runs wait rather than cancel, so an in-flight run finishes. +concurrency: + group: oats-${{ github.repository }} + cancel-in-progress: false + +jobs: + oats: + # Skip pull requests coming from forks: they must not execute untrusted + # code on the self-hosted runner. Fork PRs can be validated with a manual + # workflow_dispatch after review. + if: >- + github.event_name != 'pull_request' || + github.event.pull_request.head.repo.full_name == github.repository + runs-on: self-hosted + timeout-minutes: 1440 + steps: + - name: Checkout OATS + uses: actions/checkout@v7 + + - name: Resolve run parameters + id: params + env: + EVENT_NAME: ${{ github.event_name }} + INPUT_IMAGE: ${{ github.event.inputs.image }} + INPUT_TAG: ${{ github.event.inputs.tag }} + INPUT_GROUP: ${{ github.event.inputs.group }} + DISPATCH_IMAGE: ${{ github.event.client_payload.image }} + DISPATCH_TAG: ${{ github.event.client_payload.tag }} + run: | + set -euo pipefail + case "$EVENT_NAME" in + workflow_dispatch) + image="${INPUT_IMAGE:-opendronemap/odm}" + tag="${INPUT_TAG:-latest}" + group="${INPUT_GROUP:-all}" + ;; + repository_dispatch) + image="${DISPATCH_IMAGE:-opendronemap/odm}" + tag="${DISPATCH_TAG:-latest}" + group="all" + ;; + *) + # push / pull_request validate the harness itself: keep it fast. + image="opendronemap/odm" + tag="latest" + group="small" + ;; + esac + { + echo "image=$image" + echo "tag=$tag" + echo "group=$group" + } >> "$GITHUB_OUTPUT" + echo "Resolved image=$image tag=$tag group=$group (event=$EVENT_NAME)" + + - name: Pull ODM image + run: docker pull "${{ steps.params.outputs.image }}:${{ steps.params.outputs.tag }}" + + - name: Clear previous run results + # The self-hosted workspace persists between runs; drop old run + # directories so the newest run_manifest.json is unambiguous. Downloaded + # datasets under datasets/ are left in place to avoid re-fetching them. + run: rm -rf results/runs + + - name: Run OATS + id: run + run: | + ./run "${{ steps.params.outputs.group }}" \ + --image_name "${{ steps.params.outputs.image }}" \ + --tag "${{ steps.params.outputs.tag }}" + + - name: Locate run directory + id: rundir + if: always() + run: | + set -euo pipefail + manifest=$(find results/runs -name run_manifest.json -printf '%T@\t%p\n' 2>/dev/null \ + | sort -rn | head -1 | cut -f2-) + if [ -z "$manifest" ]; then + echo "No run_manifest.json found under results/runs" >&2 + exit 1 + fi + run_dir=$(dirname "$manifest") + echo "run_dir=$run_dir" >> "$GITHUB_OUTPUT" + echo "Run directory: $run_dir" + + - name: Publish JUnit results + if: always() && steps.rundir.outputs.run_dir != '' + uses: mikepenz/action-junit-report@v6 + with: + report_paths: ${{ steps.rundir.outputs.run_dir }}/reports/*.xml + check_name: OATS results + require_tests: true + fail_on_failure: true + + # TODO: This will need updating once the self-hosted Garage storage + # service exists For now we just store the text artifacts as the whole + # results directory is far too big to store as an artifact. + - name: Upload run artifact + if: always() && steps.rundir.outputs.run_dir != '' + uses: actions/upload-artifact@v7 + with: + name: oats-run-${{ github.run_id }} + path: | + ${{ steps.rundir.outputs.run_dir }}/run_manifest.json + ${{ steps.rundir.outputs.run_dir }}/oats_manifest.tsv + ${{ steps.rundir.outputs.run_dir }}/reports/*.xml + ${{ steps.rundir.outputs.run_dir }}/tests/*/*/task_output.txt + ${{ steps.rundir.outputs.run_dir }}/tests/*/*/log.json + ${{ steps.rundir.outputs.run_dir }}/tests/*/*/benchmark.txt + ${{ steps.rundir.outputs.run_dir }}/tests/*/*/options.json + ${{ steps.rundir.outputs.run_dir }}/tests/*/*/img_list.txt + ${{ steps.rundir.outputs.run_dir }}/tests/*/*/cameras.json + ${{ steps.rundir.outputs.run_dir }}/tests/*/*/odm_report/stats.json + ${{ steps.rundir.outputs.run_dir }}/tests/*/*/odm_report/shots.geojson + if-no-files-found: warn + retention-days: 14 diff --git a/docs/ci.md b/docs/ci.md new file mode 100644 index 0000000..3f66db3 --- /dev/null +++ b/docs/ci.md @@ -0,0 +1,70 @@ +# CI + +`.github/workflows/oats.yml` runs the suite on a self-hosted runner: on demand, +when OATS master changes, and when ODM publishes a new image. The workflow +itself is the reference for what runs when — this covers the parts that live +outside it, namely the runner, the cross-repo token and the ODM-side trigger. + +## The runner + +Register a runner against this repository (*Settings → Actions → Runners*). The +default `self-hosted` label is all the workflow asks for. + +It needs docker, plus `git`, `wget`, `rsync`, `sed`, `unzip`, `jq`, and +`gdalinfo` on `PATH`; `./run` bootstraps bats itself. Give it plenty of RAM — +the suite is RAM-bound and the `all` group sets the ceiling. Each full run +generates about 60GB of output right now so as a rough guide we should aim for +about 1TB of disk space. + +Keep a single runner attached. The workflow serialises runs with a `concurrency` +group, but that only helps if there is one machine to serialise onto — a second +runner would happily pick up a queued run alongside the first. + +## No GPU + +I assume the runner has no GPU, so the suite only exercises ODM's CPU path, and +`opendronemap/odm:gpu` is not tested at all — `publish-docker-gpu.yaml` has no +dispatch step by design. Testing it would need GPU hardware with +`nvidia-container-toolkit`, a `--gpus` flag in the harness's `docker run` (it +passes none today), and the dispatch step added on the ODM side. + +## The ODM trigger + +ODM's `publish-docker.yaml` dispatches to OATS after pushing a new image. The +payload contract is at the top of `oats.yml`; the ODM-side step is: + +```yaml + - name: Trigger OATS test suite + run: | + curl -sSf -X POST \ + -H "Accept: application/vnd.github+json" \ + -H "Authorization: Bearer ${{ secrets.OATS_DISPATCH_TOKEN }}" \ + https://api.github.com/repos/OpenDroneMap/oats/dispatches \ + -d '{"event_type":"odm-image-published","client_payload":{"image":"opendronemap/odm","tag":"latest"}}' +``` + +`OATS_DISPATCH_TOKEN` is a secret on the **ODM** repository, not this one. The +`GITHUB_TOKEN` Actions provides will not do: it is scoped to the repository +running the workflow, and events raised with it never start a workflow run. + +To generate one, from an account with admin on this repo: + +1. *Settings → Developer settings → Personal access tokens → Fine-grained + tokens → Generate new token*. +2. Resource owner **OpenDroneMap**. Org-owned tokens need an org owner to + approve the request before they work. +3. Repository access: *Only select repositories* → `OpenDroneMap/oats`. +4. Repository permissions: **Contents → Read and write**. Broader than we want, + but `repository_dispatch` has no permission of its own and GitHub files it + under Contents. +5. Add it to ODM under *Settings → Secrets and variables → Actions → New + repository secret*, named `OATS_DISPATCH_TOKEN`. + +Fine-grained tokens expire, so note the date. When one lapses the dispatch step +fails and the ODM publish job goes red. + +## Results + +Currently we store the text outputs as a github artifact which will allow us to +review any failures. This will be replaced by the s3-compatible Garage server +once its set up.